From d46f449e4baf3d19d27a05de44041cfc6cf69949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Ml=C3=A1dek?= Date: Tue, 18 Jan 2022 17:05:45 +0100 Subject: [PATCH] file upload todo: - don't return file hash in ListingResult, solve this marking problem in general? - work around persisterror across filesystems - add proper BLOB metadata --- Cargo.lock | 35 ++++++ Cargo.toml | 1 + src/routes.rs | 118 +++++++++++++++----- webui/src/components/AddModal.svelte | 124 ++++++++++++++++++++- webui/src/components/layout/Header.svelte | 7 +- webui/src/components/utils/Selector.svelte | 2 +- 6 files changed, 251 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9441042..7859eeb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,6 +149,24 @@ dependencies = [ "syn", ] +[[package]] +name = "actix-multipart" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774bfeb11b54bf9c857a005b8ab893293da4eaff79261a66a9200dab7f5ab6e3" +dependencies = [ + "actix-service", + "actix-utils", + "actix-web", + "bytes 0.5.6", + "derive_more", + "futures-util", + "httparse", + "log", + "mime", + "twoway", +] + [[package]] name = "actix-router" version = "0.2.7" @@ -2356,12 +2374,28 @@ dependencies = [ "trust-dns-proto", ] +[[package]] +name = "twoway" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57ffb460d7c24cd6eda43694110189030a3d1dfe418416d9468fd1c1d290b47" +dependencies = [ + "memchr", + "unchecked-index", +] + [[package]] name = "typenum" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "unchecked-index" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c" + [[package]] name = "unicase" version = "2.6.0" @@ -2416,6 +2450,7 @@ version = "0.0.16" dependencies = [ "actix", "actix-files", + "actix-multipart", "actix-rt 2.5.0", "actix-web", "actix_derive", diff --git a/Cargo.toml b/Cargo.toml index 79a6bb5..3a633a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ is_executable = { version = "1.0.1", optional = true } webbrowser = { version = "^0.5.5", optional = true } nonempty = "0.6.0" +actix-multipart = "0.3.0" [features] default = ["desktop", "previews"] diff --git a/src/routes.rs b/src/routes.rs index 8cbd8aa..990ee0e 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -1,24 +1,31 @@ use crate::addressing::{Address, Addressable}; use crate::database::entry::{Entry, InEntry}; use crate::database::hierarchies::{list_roots, resolve_path, UHierPath}; +use crate::database::inner::models; use crate::database::lang::Query; use crate::database::UpEndDatabase; use crate::previews::PreviewStore; -use crate::util::hash::{decode, encode}; +use crate::util::hash::{decode, encode, Hashable}; use crate::util::jobs::JobContainer; use actix_files::NamedFile; +use actix_multipart::Multipart; use actix_web::error::{ErrorBadRequest, ErrorInternalServerError, ErrorNotFound}; use actix_web::http; use actix_web::{delete, error, get, post, put, web, Either, Error, HttpResponse}; -use anyhow::anyhow; -use anyhow::Result; -use futures_util::StreamExt; +use anyhow::{anyhow, Result}; +use chrono::{NaiveDateTime, Utc}; +use futures_util::TryStreamExt; use log::{debug, info, trace}; use serde::Deserialize; use serde_json::json; use std::collections::HashMap; use std::convert::TryFrom; +use std::fs; +use std::io::Write; +use std::path::PathBuf; use std::sync::{Arc, RwLock}; +use std::time::UNIX_EPOCH; +use tempfile::NamedTempFile; #[cfg(feature = "desktop")] use is_executable::IsExecutable; @@ -178,19 +185,86 @@ pub async fn get_object( #[put("/api/obj")] pub async fn put_object( state: web::Data, - mut payload: web::Payload, + payload: Either, Multipart>, ) -> Result { - let body = load_body(&mut payload) - .await - .map_err(error::ErrorBadRequest)?; - let in_entry = serde_json::from_slice::(&body).map_err(ErrorBadRequest)?; - let entry = Entry::try_from(in_entry).map_err(ErrorInternalServerError)?; + let (result_address, entry) = match payload { + Either::A(in_entry) => { + let in_entry = in_entry.into_inner(); + let entry = Entry::try_from(in_entry).map_err(ErrorInternalServerError)?; - let connection = state.upend.connection().map_err(ErrorInternalServerError)?; + let connection = state.upend.connection().map_err(ErrorInternalServerError)?; - let result_address = connection - .insert_entry(entry.clone()) - .map_err(ErrorInternalServerError)?; + Ok(( + connection + .insert_entry(entry.clone()) + .map_err(ErrorInternalServerError)?, + Some(entry), + )) + } + Either::B(mut multipart) => { + if let Some(mut field) = multipart.try_next().await? { + let content_disposition = field + .content_disposition() + .ok_or_else(|| HttpResponse::BadRequest().finish())?; + + let filename = content_disposition.get_filename(); + + let mut file = NamedTempFile::new()?; + while let Some(chunk) = field.try_next().await? { + file = web::block(move || file.write_all(&chunk).map(|_| file)).await?; + } + let path = PathBuf::from(file.path()); + let hash = web::block(move || path.hash()).await?; + + let address = Address::Hash(hash.clone()); + + let addr_str = encode(address.encode().map_err(ErrorInternalServerError)?); + let final_name = if let Some(filename) = filename { + format!("{addr_str}_{filename}") + } else { + addr_str + }; + + let final_path = state.upend.vault_path.join(&final_name); + + file.persist(&final_path) + .map_err(ErrorInternalServerError)?; + + let metadata = fs::metadata(&final_path)?; + let size = metadata.len() as i64; + let mtime = metadata + .modified() + .map(|t| { + NaiveDateTime::from_timestamp( + t.duration_since(UNIX_EPOCH).unwrap().as_secs() as i64, + 0, + ) + }) + .ok(); + + let connection = state.upend.connection().map_err(ErrorInternalServerError)?; + + connection + .insert_file(models::NewFile { + path: final_name, + hash: hash.0, + added: NaiveDateTime::from_timestamp(Utc::now().timestamp(), 0), + size, + mtime, + }) + .map_err(ErrorInternalServerError)?; + + if let Some(filename) = filename { + upend_insert_val!(&connection, address, "LBL", filename) + .map_err(ErrorInternalServerError)?; + } + + Ok((address, None)) + } else { + Err(ErrorBadRequest(anyhow!("wat"))) + } + } + }?; Ok(HttpResponse::Ok().json( [( @@ -199,7 +273,7 @@ pub async fn put_object( )] .iter() .cloned() - .collect::>(), + .collect::>>(), )) } @@ -376,17 +450,3 @@ pub async fn get_thumbnail( } Err(error::ErrorNotImplemented("Previews not enabled.")) } - -const MAX_BODY_SIZE: usize = 1_000_000; - -async fn load_body(payload: &mut web::Payload) -> Result> { - let mut body = web::BytesMut::new(); - while let Some(chunk) = payload.next().await { - let chunk = chunk?; - if (body.len() + chunk.len()) > MAX_BODY_SIZE { - return Err(anyhow!("OVERFLOW")); - } - body.extend_from_slice(&chunk); - } - Ok(body.as_ref().to_vec()) -} diff --git a/webui/src/components/AddModal.svelte b/webui/src/components/AddModal.svelte index 272dd1a..f1a5fca 100644 --- a/webui/src/components/AddModal.svelte +++ b/webui/src/components/AddModal.svelte @@ -8,21 +8,76 @@ -
-
- {#each files as file} -
{file.name}
- {/each} +
+
+
+ {#each files as file} +
+
+ +
+
{file.name}
+
+ {/each} +
+
+ +
@@ -40,5 +95,64 @@ &.visible { display: unset; } + + &.uploading { + cursor: progress; + + .addmodal { + filter: brightness(0.5); + } + } + } + + .addmodal { + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + + background: var(--background); + + color: var(--foreground); + border: solid 2px var(--foreground); + border-radius: 8px; + padding: 1rem; + } + + .files { + display: flex; + flex-direction: column; + gap: 1em; + + padding: 0.5em; + + overflow-y: auto; + max-height: 66vh; + } + + .file { + display: flex; + align-items: center; + + border: 1px solid var(--foreground); + border-radius: 4px; + background: var(--background-lighter); + padding: 0.5em; + + .icon { + font-size: 24px; + } + + .label { + flex-grow: 1; + text-align: center; + } + } + + .controls { + display: flex; + justify-content: center; + font-size: 48px; + margin-top: 0.5rem; } diff --git a/webui/src/components/layout/Header.svelte b/webui/src/components/layout/Header.svelte index 4055234..73b4601 100644 --- a/webui/src/components/layout/Header.svelte +++ b/webui/src/components/layout/Header.svelte @@ -47,7 +47,12 @@
diff --git a/webui/src/components/utils/Selector.svelte b/webui/src/components/utils/Selector.svelte index efd5d64..0d8dfba 100644 --- a/webui/src/components/utils/Selector.svelte +++ b/webui/src/components/utils/Selector.svelte @@ -71,7 +71,7 @@
(inputFocused = ev.detail)} />