upend/cli/src/routes.rs

842 lines
28 KiB
Rust

use crate::extractors::{self};
use crate::previews::PreviewStore;
use crate::util::exec::block_background;
use actix_files::NamedFile;
use actix_multipart::Multipart;
use actix_web::error::{
ErrorBadRequest, ErrorInternalServerError, ErrorNotFound, ErrorUnauthorized,
};
use actix_web::{delete, error, get, post, put, web, Either, Error, HttpResponse};
use actix_web::{http, Responder};
use actix_web::{
http::header::{CacheControl, CacheDirective, ContentDisposition, DispositionType},
HttpRequest,
};
use anyhow::{anyhow, Result};
use futures::channel::oneshot;
use futures_util::TryStreamExt;
use serde::{Deserialize, Serialize};
use serde_json::json;
use std::collections::HashMap;
use std::convert::{TryFrom, TryInto};
use std::io::Write;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tempfile::NamedTempFile;
use tracing::{debug, info, trace};
use upend::addressing::{Address, Addressable};
use upend::common::build;
use upend::config::UpEndConfig;
use upend::database::constants::{ADDED_ATTR, LABEL_ATTR};
use upend::database::entry::{Entry, EntryValue, InvariantEntry};
use upend::database::hierarchies::{list_roots, resolve_path, UHierPath};
use upend::database::lang::Query;
use upend::database::stores::{Blob, UpStore};
use upend::database::UpEndDatabase;
use upend::util::hash::{b58_decode, b58_encode};
use upend::util::jobs;
use url::Url;
use uuid::Uuid;
#[cfg(feature = "desktop")]
use is_executable::IsExecutable;
#[derive(Clone)]
pub struct State {
pub upend: Arc<UpEndDatabase>,
pub store: Arc<Box<dyn UpStore + Sync + Send>>,
pub config: UpEndConfig,
pub job_container: jobs::JobContainer,
pub preview_store: Option<Arc<PreviewStore>>,
pub preview_pool: Option<Arc<rayon::ThreadPool>>,
}
#[derive(Debug, Serialize, Deserialize)]
struct JwtClaims {
exp: usize,
}
#[derive(Deserialize)]
pub struct LoginRequest {
key: String,
}
#[post("/api/auth/login")]
pub async fn login(
state: web::Data<State>,
payload: web::Json<LoginRequest>,
) -> Result<HttpResponse, Error> {
if state.config.key.is_none() || Some(&payload.key) == state.config.key.as_ref() {
let claims = JwtClaims {
exp: (SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(ErrorInternalServerError)?
.as_secs()
+ 7 * 24 * 60 * 60) as usize,
};
let token = jsonwebtoken::encode(
&jsonwebtoken::Header::default(),
&claims,
&jsonwebtoken::EncodingKey::from_secret(state.config.secret.as_ref()),
)
.map_err(ErrorInternalServerError)?;
Ok(HttpResponse::Ok().json(json!({ "token": token })))
} else {
Err(ErrorUnauthorized("Incorrect token."))
}
}
fn check_auth(req: &HttpRequest, state: &State) -> Result<(), actix_web::Error> {
if let Some(key) = &state.config.key {
if let Some(auth_header) = req.headers().get("Authorization") {
let auth_header = auth_header.to_str().map_err(|err| {
ErrorBadRequest(format!("Invalid value in Authorization header: {err:?}"))
})?;
let token = jsonwebtoken::decode::<JwtClaims>(
auth_header,
&jsonwebtoken::DecodingKey::from_secret(key.as_ref()),
&jsonwebtoken::Validation::default(),
);
token
.map(|_| ())
.map_err(|err| ErrorUnauthorized(format!("Invalid token: {err:?}")))
} else {
Err(ErrorUnauthorized("Authorization required."))
}
} else {
Ok(())
}
}
#[derive(Deserialize)]
pub struct RawRequest {
native: Option<String>,
inline: Option<String>,
}
#[get("/api/raw/{hash}")]
pub async fn get_raw(
state: web::Data<State>,
web::Query(query): web::Query<RawRequest>,
hash: web::Path<String>,
) -> Result<impl Responder, Error> {
let address =
Address::decode(&b58_decode(hash.into_inner()).map_err(ErrorInternalServerError)?)
.map_err(ErrorInternalServerError)?;
if let Address::Hash(hash) = address {
let hash = Arc::new(hash);
let _hash = hash.clone();
let _store = state.store.clone();
let blobs = web::block(move || _store.retrieve(_hash.as_ref()))
.await
.map_err(ErrorInternalServerError)?;
if let Some(blob) = blobs.get(0) {
let file_path = blob.get_file_path();
if query.native.is_none() {
return Ok(Either::A(
NamedFile::open(file_path)?
.set_content_disposition(ContentDisposition {
disposition: if query.inline.is_some() {
DispositionType::Inline
} else {
DispositionType::Attachment
},
parameters: vec![],
})
.with_header(
http::header::CACHE_CONTROL,
CacheControl(vec![
CacheDirective::MaxAge(2678400),
CacheDirective::Extension("immutable".into(), None),
]),
),
));
} else if state.config.desktop_enabled {
#[cfg(feature = "desktop")]
{
info!("Opening {:?}...", file_path);
let mut response = HttpResponse::NoContent();
let path = if !file_path.is_executable() || state.config.trust_executables {
file_path
} else {
response
.header(
http::header::WARNING,
"199 - Opening parent directory due to file being executable.",
)
.header(
http::header::ACCESS_CONTROL_EXPOSE_HEADERS,
http::header::WARNING.to_string(),
);
file_path.parent().ok_or_else(|| {
ErrorInternalServerError("No parent to open as fallback.")
})?
};
opener::open(path).map_err(error::ErrorServiceUnavailable)?;
return Ok(Either::B(response.finish()));
}
#[cfg(not(feature = "desktop"))]
unreachable!()
} else {
return Err(error::ErrorNotImplemented("Desktop features not enabled."));
}
}
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
let _hash = hash.clone();
let entry = web::block(move || connection.retrieve_entry(_hash.as_ref()))
.await
.map_err(ErrorInternalServerError)?;
if let Some(entry) = entry {
return Ok(Either::B(HttpResponse::Ok().json(entry)));
}
Err(error::ErrorNotFound("NOT FOUND"))
} else {
Err(ErrorBadRequest(
"Address does not refer to a rawable object.",
))
}
}
#[get("/api/thumb/{hash}")]
pub async fn get_thumbnail(
state: web::Data<State>,
hash: web::Path<String>,
web::Query(query): web::Query<HashMap<String, String>>,
) -> Result<Either<NamedFile, HttpResponse>, Error> {
#[cfg(feature = "previews")]
if let Some(preview_store) = &state.preview_store {
let hash = hash.into_inner();
let address = Address::decode(&b58_decode(&hash).map_err(ErrorInternalServerError)?)
.map_err(ErrorInternalServerError)?;
if let Address::Hash(address_hash) = address {
let preview_store = preview_store.clone();
let (tx, rx) = oneshot::channel();
let _job_container = state.job_container.clone();
state.preview_pool.as_ref().unwrap().spawn(move || {
let result = preview_store.get(address_hash, query, _job_container);
tx.send(result).unwrap();
});
let preview_result = rx.await.unwrap().map_err(ErrorInternalServerError)?;
if let Some(preview_path) = preview_result {
let mut file = NamedFile::open(&preview_path)?.disable_content_disposition();
if let Some(mime_type) = tree_magic_mini::from_filepath(&preview_path) {
if let Ok(mime) = mime_type.parse() {
file = file.set_content_type(mime);
}
}
return Ok(Either::A(file));
} else {
return Ok(Either::B(
HttpResponse::SeeOther()
.header(http::header::LOCATION, format!("../../api/raw/{hash}"))
.finish(),
));
}
} else {
return Err(ErrorBadRequest(
"Address does not refer to a previewable object.",
));
}
}
Err(error::ErrorNotImplemented("Previews not enabled."))
}
#[post("/api/query")]
pub async fn get_query(state: web::Data<State>, query: String) -> Result<HttpResponse, Error> {
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
let in_query: Query = query.parse().map_err(ErrorBadRequest)?;
let entries = web::block(move || connection.query(in_query))
.await
.map_err(ErrorInternalServerError)?;
let mut result: HashMap<String, Entry> = HashMap::new();
for entry in entries {
result.insert(
b58_encode(
entry
.address()
.map_err(ErrorInternalServerError)?
.encode()
.map_err(ErrorInternalServerError)?,
),
entry,
);
}
Ok(HttpResponse::Ok().json(&result))
}
trait EntriesAsHash {
fn as_hash(&self) -> Result<HashMap<String, &Entry>>;
}
impl EntriesAsHash for Vec<Entry> {
fn as_hash(&self) -> Result<HashMap<String, &Entry>> {
let mut result: HashMap<String, &Entry> = HashMap::new();
for entry in self {
result.insert(b58_encode(entry.address()?.encode()?), entry);
}
Ok(result)
}
}
#[get("/api/obj/{address_str}")]
pub async fn get_object(
state: web::Data<State>,
address: web::Path<Address>,
) -> Result<HttpResponse, Error> {
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
let address = address.into_inner();
let _address = address.clone();
let result: Vec<Entry> = web::block(move || connection.retrieve_object(&_address))
.await
.map_err(ErrorInternalServerError)?;
debug!("{:?}", result);
// TODO: make this automatically derive from `Address` definition
let (entity_type, entity_content) = match address {
Address::Hash(_) => ("Hash", None),
Address::Uuid(_) => ("Uuid", None),
Address::Attribute(attribute) => ("Attribute", Some(attribute)),
Address::Url(url) => ("Url", Some(url.to_string())),
};
Ok(HttpResponse::Ok().json(json!({
"entity": {
"t": entity_type,
"c": entity_content
},
"entries": result.as_hash().map_err(ErrorInternalServerError)?
})))
}
#[derive(Debug, Clone, Deserialize)]
#[serde(untagged)]
pub enum InAddress {
Address(String),
Components { t: String, c: Option<String> },
}
impl TryInto<Address> for InAddress {
type Error = anyhow::Error;
fn try_into(self) -> Result<Address, Self::Error> {
Ok(match self {
InAddress::Address(address) => address.parse()?,
InAddress::Components { t, c } => {
// I absolutely cannot handle serde magic right now
// TODO: make this automatically derive from `Address` definition
match t.as_str() {
"Attribute" => Address::Attribute(c.ok_or(anyhow!("Missing attribute."))?),
"Url" => Address::Url(if let Some(string) = c {
Url::parse(&string)?
} else {
Err(anyhow!("Missing URL."))?
}),
"Uuid" => match c {
Some(c) => c.parse()?,
None => Address::Uuid(Uuid::new_v4()),
},
_ => c.ok_or(anyhow!("Missing address."))?.parse()?,
}
}
})
}
}
#[derive(Debug, Clone, Deserialize)]
pub struct InEntry {
pub entity: Option<InAddress>,
pub attribute: String,
pub value: EntryValue,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(untagged, deny_unknown_fields)]
pub enum PutInput {
Entry(InEntry),
EntryList(Vec<InEntry>),
Address { entity: InAddress },
}
#[derive(Deserialize)]
pub struct UpdateQuery {
provenance: Option<String>,
}
#[put("/api/obj")]
pub async fn put_object(
req: HttpRequest,
state: web::Data<State>,
payload: web::Json<PutInput>,
web::Query(query): web::Query<UpdateQuery>,
) -> Result<HttpResponse, Error> {
check_auth(&req, &state)?;
let (entry_address, entity_address) = {
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
let in_entry = payload.into_inner();
debug!("PUTting {in_entry:?}");
let provenance = query.provenance.clone();
let process_inentry = move |in_entry: InEntry| -> Result<Entry> {
if let Some(entity) = in_entry.entity {
Ok(Entry {
entity: entity.try_into()?,
attribute: in_entry.attribute,
value: in_entry.value,
provenance: match &provenance {
Some(s) => format!("API {}", s),
None => "API".to_string(),
},
timestamp: chrono::Utc::now().naive_utc(),
})
} else {
Entry::try_from(&InvariantEntry {
attribute: in_entry.attribute,
value: in_entry.value,
})
}
};
match in_entry {
PutInput::Entry(in_entry) => {
let entry = process_inentry(in_entry).map_err(ErrorBadRequest)?;
web::block::<_, _, anyhow::Error>(move || {
Ok((
Some(connection.insert_entry(entry.clone())?),
Some(entry.entity),
))
})
.await
.map_err(ErrorInternalServerError)?
}
PutInput::EntryList(entries) => {
web::block(move || {
connection.transaction::<_, anyhow::Error, _>(|| {
for entry in entries {
connection.insert_entry(process_inentry(entry)?)?;
}
Ok(())
})
})
.await
.map_err(ErrorBadRequest)?;
(None, None)
}
PutInput::Address { entity: in_address } => {
let address = in_address.try_into().map_err(ErrorBadRequest)?;
let label_entry = match &address {
Address::Hash(_) | Address::Uuid(_) => None,
Address::Attribute(_) => None,
Address::Url(url) => Some(Entry {
entity: address.clone(),
attribute: LABEL_ATTR.to_string(),
value: url.clone().into(),
provenance: match &query.provenance {
Some(s) => format!("API {}", s),
None => "API".to_string(),
},
timestamp: chrono::Utc::now().naive_utc(),
}),
};
let _address = address.clone();
let _job_container = state.job_container.clone();
let _store = state.store.clone();
block_background::<_, _, anyhow::Error>(move || {
let entry_count =
extractors::extract(&_address, &connection, _store, _job_container);
debug!("Added {entry_count} extracted entries for {_address:?}");
Ok(())
});
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
web::block(move || {
connection.transaction::<_, anyhow::Error, _>(|| {
if connection.retrieve_object(&address)?.is_empty() {
connection.insert_entry(Entry {
entity: address.clone(),
attribute: ADDED_ATTR.to_string(),
value: EntryValue::Number(
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs() as f64,
),
provenance: match &query.provenance {
Some(s) => format!("API {}", s),
None => "API".to_string(),
},
timestamp: chrono::Utc::now().naive_utc(),
})?;
}
if let Some(label_entry) = label_entry {
connection.insert_entry(label_entry)?;
}
Ok((None, Some(address)))
})
})
.await
.map_err(ErrorInternalServerError)?
}
}
};
Ok(HttpResponse::Ok().json([entry_address, entity_address]))
}
#[put("/api/blob")]
pub async fn put_blob(
req: HttpRequest,
state: web::Data<State>,
mut payload: Multipart,
) -> Result<HttpResponse, Error> {
check_auth(&req, &state)?;
if let Some(mut field) = payload.try_next().await? {
let content_disposition = field
.content_disposition()
.ok_or_else(|| HttpResponse::BadRequest().finish())?;
let filename = content_disposition.get_filename().map(String::from);
let mut file = NamedTempFile::new()?;
while let Some(chunk) = field.try_next().await? {
file = web::block(move || file.write_all(&chunk).map(|_| file)).await?;
}
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
let _store = state.store.clone();
let _filename = filename.clone();
let hash = web::block(move || {
_store.store(connection, Blob::from_filepath(file.path()), _filename)
})
.await
.map_err(ErrorInternalServerError)?;
let address = Address::Hash(hash);
if let Some(ref filename) = filename {
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
let _address = address.clone();
let _filename = filename.clone();
let _ =
web::block(move || upend_insert_val!(&connection, _address, LABEL_ATTR, _filename))
.await;
}
let _address = address.clone();
let _job_container = state.job_container.clone();
let _store = state.store.clone();
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
block_background::<_, _, anyhow::Error>(move || {
let entry_count = extractors::extract(&_address, &connection, _store, _job_container);
debug!("Added {entry_count} extracted entries for {_address:?}");
Ok(())
});
Ok(HttpResponse::Ok().json(address))
} else {
Err(ErrorBadRequest("Multipart contains no fields."))
}
}
#[put("/api/obj/{address}/{attribute}")]
pub async fn put_object_attribute(
req: HttpRequest,
state: web::Data<State>,
web::Path((address, attribute)): web::Path<(Address, String)>,
value: web::Json<EntryValue>,
web::Query(query): web::Query<UpdateQuery>,
) -> Result<HttpResponse, Error> {
check_auth(&req, &state)?;
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
let new_address = web::block(move || {
connection.transaction::<_, anyhow::Error, _>(|| {
let existing_attr_entries =
connection.query(format!(r#"(matches @{address} "{attribute}" ?)"#).parse()?)?;
for eae in existing_attr_entries {
let _ = connection.remove_object(eae.address()?)?;
}
let new_attr_entry = Entry {
entity: address,
attribute,
value: value.into_inner(),
provenance: match &query.provenance {
Some(s) => format!("API {}", s),
None => "API".to_string(),
},
timestamp: chrono::Utc::now().naive_utc(),
};
connection.insert_entry(new_attr_entry)
})
})
.await
.map_err(ErrorInternalServerError)?;
Ok(HttpResponse::Ok().json(new_address))
}
#[delete("/api/obj/{address_str}")]
pub async fn delete_object(
req: HttpRequest,
state: web::Data<State>,
address: web::Path<Address>,
) -> Result<HttpResponse, Error> {
check_auth(&req, &state)?;
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
let _ = web::block(move || connection.remove_object(address.into_inner()))
.await
.map_err(ErrorInternalServerError)?;
Ok(HttpResponse::Ok().finish())
}
// #[post("api/obj/{address_str}")]
// pub async fn update_attribute(
// state: web::Data<State>,
// address_str: web::Path<String>,
// mut payload: web::Payload,
// ) -> Result<HttpResponse, Error> {
// let body = load_body(&mut payload)
// .await
// .map_err(error::ErrorBadRequest)?;
// let entry_value = serde_json::from_slice::<EntryValue>(&body).map_err(ErrorBadRequest)?;
// let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
// connection
// .transaction::<_, anyhow::Error, _>(|| {
// let address = Address::decode(&decode(address_str.into_inner())?)?;
// let _ = connection.remove_object(address)?;
// Ok(())
// })
// .map_err(ErrorInternalServerError)?;
// Ok(HttpResponse::Ok().finish())
// }
#[derive(Deserialize)]
pub struct GetAddressRequest {
attribute: Option<String>,
// url: Option<String>,
}
#[get("/api/address")]
pub async fn get_address(
web::Query(query): web::Query<GetAddressRequest>,
) -> Result<HttpResponse, Error> {
let address = match query {
GetAddressRequest {
attribute: Some(attribute),
} => Address::Attribute(attribute),
_ => Err(ErrorBadRequest("Specify one of: `attribute`"))?,
};
Ok(HttpResponse::Ok()
.set_header(
http::header::CACHE_CONTROL,
CacheControl(vec![
CacheDirective::MaxAge(2678400),
CacheDirective::Extension("immutable".into(), None),
]),
)
.json(format!("{}", address)))
}
#[get("/api/all/attributes")]
pub async fn get_all_attributes(state: web::Data<State>) -> Result<HttpResponse, Error> {
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
let attributes = web::block(move || connection.get_all_attributes())
.await
.map_err(ErrorInternalServerError)?;
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
let result: serde_json::Value = attributes
.into_iter()
.map(|attribute| {
json!({
"name": attribute,
"labels": connection
.retrieve_object(&Address::Attribute(attribute))
.unwrap_or_else(|_| vec![])
.into_iter()
.filter_map(|e| {
if e.attribute == LABEL_ATTR {
if let EntryValue::String(label) = e.value {
Some(label)
} else {
None
}
} else {
None
}
})
.collect::<Vec<String>>(),
})
})
.collect();
Ok(HttpResponse::Ok().json(result))
}
#[get("/api/hier/{path:.*}")]
pub async fn list_hier(
state: web::Data<State>,
path: web::Path<String>,
) -> Result<HttpResponse, Error> {
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
if path.is_empty() {
Ok(HttpResponse::MovedPermanently()
.header(http::header::LOCATION, "../../api/hier_roots")
.finish())
} else {
let upath: UHierPath = path.into_inner().parse().map_err(ErrorBadRequest)?;
trace!(r#"Listing path "{}""#, upath);
// todo: 500 if actual error occurs
let path = web::block(move || resolve_path(&connection, &upath, false))
.await
.map_err(ErrorNotFound)?;
match path.last() {
Some(addr) => Ok(HttpResponse::Found()
.header(http::header::LOCATION, format!("../../api/obj/{}", addr))
.finish()),
None => Ok(HttpResponse::NotFound().finish()),
}
}
}
#[get("/api/hier_roots")]
pub async fn list_hier_roots(state: web::Data<State>) -> Result<HttpResponse, Error> {
let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
let result = web::block(move || {
list_roots(&connection)?
.into_iter()
.map(|root| connection.retrieve_object(&root))
.collect::<Result<Vec<Vec<Entry>>>>()
})
.await
.map_err(ErrorInternalServerError)?
.concat();
Ok(HttpResponse::Ok().json(result.as_hash().map_err(ErrorInternalServerError)?))
}
// #[derive(Deserialize)]
// pub struct RescanRequest {
// full: Option<String>,
// }
#[post("/api/refresh")]
pub async fn api_refresh(
req: HttpRequest,
state: web::Data<State>,
// web::Query(query): web::Query<RescanRequest>,
) -> Result<HttpResponse, Error> {
check_auth(&req, &state)?;
block_background::<_, _, anyhow::Error>(move || {
let _ = state
.store
.update(&state.upend, state.job_container.clone(), false);
let _ = crate::extractors::extract_all(
state.upend.clone(),
state.store.clone(),
state.job_container.clone(),
);
Ok(())
});
Ok(HttpResponse::Ok().finish())
}
#[get("/api/store")]
pub async fn store_info(state: web::Data<State>) -> Result<HttpResponse, Error> {
Ok(HttpResponse::Ok().json(json!({
"main": state.store.stats().map_err(ErrorInternalServerError)?
})))
}
#[derive(Deserialize)]
pub struct JobsRequest {
full: Option<String>,
}
#[get("/api/jobs")]
pub async fn get_jobs(
state: web::Data<State>,
web::Query(query): web::Query<JobsRequest>,
) -> Result<HttpResponse, Error> {
let jobs = state
.job_container
.get_jobs()
.map_err(ErrorInternalServerError)?;
Ok(HttpResponse::Ok().json(if query.full.is_some() {
jobs
} else {
jobs.into_iter()
.filter(|(_, j)| matches!(j.state, jobs::JobState::InProgress))
.collect()
}))
}
#[get("/api/info")]
pub async fn get_info(state: web::Data<State>) -> Result<HttpResponse, Error> {
Ok(HttpResponse::Ok().json(json!({
"name": state.config.vault_name,
// "location": &*state.store.path,
"version": build::PKG_VERSION,
"desktop": state.config.desktop_enabled
})))
}
#[cfg(test)]
mod tests {
use super::*;
use anyhow::Result;
#[test]
fn test_in_address() -> Result<()> {
let address = Address::Url(Url::parse("https://upend.dev").unwrap());
let in_address = InAddress::Address(address.to_string());
assert_eq!(address, in_address.try_into()?);
let in_address = InAddress::Components {
t: "Url".into(),
c: Some("https://upend.dev".into()),
};
assert_eq!(address, in_address.try_into()?);
Ok(())
}
}