fix: limit previews to NUM_CPU/2 at a time, avoid brown lock-ups
parent
d671640c04
commit
0bb4639859
|
|
@ -1064,12 +1064,13 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.19"
|
||||
version = "0.3.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28560757fe2bb34e79f907794bb6b22ae8b0e5c669b638a1132f2592b19035b4"
|
||||
checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-executor",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
|
|
@ -1078,9 +1079,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.19"
|
||||
version = "0.3.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba3dda0b6588335f360afc675d0564c17a77a2bda81ca178a4b6081bd86c7f0b"
|
||||
checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
|
|
@ -1088,21 +1089,32 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.19"
|
||||
version = "0.3.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0c8ff0461b82559810cdccfde3215c3f373807f5e5232b71479bff7bb2583d7"
|
||||
checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.19"
|
||||
version = "0.3.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f9d34af5a1aac6fb380f735fe510746c38067c5bf16c7fd250280503c971b2"
|
||||
checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.19"
|
||||
version = "0.3.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6dbd947adfffb0efc70599b3ddcf7b5597bb5fa9e245eb99f62b3a5f7bb8bd3c"
|
||||
checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -1111,21 +1123,21 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.19"
|
||||
version = "0.3.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3055baccb68d74ff6480350f8d6eb8fcfa3aa11bdc1a1ae3afdd0514617d508"
|
||||
checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.19"
|
||||
version = "0.3.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ee7c6485c30167ce4dfb83ac568a849fe53274c831081476ee13e0dce1aad72"
|
||||
checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.19"
|
||||
version = "0.3.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9b5cf40b47a271f77a8b1bec03ca09044d99d2372c0de244e66430761127164"
|
||||
checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
|
|
@ -3225,6 +3237,7 @@ dependencies = [
|
|||
"diesel_migrations",
|
||||
"dotenv",
|
||||
"filebuffer",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"id3",
|
||||
"image",
|
||||
|
|
@ -3240,6 +3253,7 @@ dependencies = [
|
|||
"multibase",
|
||||
"multihash",
|
||||
"nonempty",
|
||||
"num_cpus",
|
||||
"once_cell",
|
||||
"opener",
|
||||
"rand 0.8.4",
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ anyhow = "1.0"
|
|||
thiserror = "1.0"
|
||||
|
||||
rayon = "1.4.0"
|
||||
num_cpus = "1.13"
|
||||
futures = "0.3.24"
|
||||
futures-util = "~0.3.12"
|
||||
lazy_static = "1.4.0"
|
||||
once_cell = "1.7.2"
|
||||
|
|
|
|||
10
src/main.rs
10
src/main.rs
|
|
@ -151,6 +151,13 @@ fn main() -> Result<()> {
|
|||
preview_path.clone(),
|
||||
store.clone(),
|
||||
)));
|
||||
#[cfg(feature = "previews")]
|
||||
let preview_pool = Some(Arc::new(
|
||||
rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(num_cpus::get() / 2)
|
||||
.build()
|
||||
.unwrap(),
|
||||
));
|
||||
|
||||
if matches.is_present("CLEAN") {
|
||||
info!("Cleaning temporary directories...");
|
||||
|
|
@ -164,6 +171,8 @@ fn main() -> Result<()> {
|
|||
|
||||
#[cfg(not(feature = "previews"))]
|
||||
let preview_store = None;
|
||||
#[cfg(not(feature = "previews"))]
|
||||
let preview_pool = None;
|
||||
|
||||
let mut bind: SocketAddr = matches
|
||||
.value_of("BIND")
|
||||
|
|
@ -204,6 +213,7 @@ fn main() -> Result<()> {
|
|||
store,
|
||||
job_container: job_container.clone(),
|
||||
preview_store,
|
||||
preview_pool,
|
||||
desktop_enabled,
|
||||
secret,
|
||||
key,
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ use actix_web::{
|
|||
HttpRequest,
|
||||
};
|
||||
use anyhow::{anyhow, Result};
|
||||
use futures::channel::oneshot;
|
||||
use futures_util::TryStreamExt;
|
||||
use log::{debug, info, trace, warn};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
|
@ -44,6 +45,7 @@ pub struct State {
|
|||
pub vault_name: Option<String>,
|
||||
pub job_container: JobContainer,
|
||||
pub preview_store: Option<Arc<PreviewStore>>,
|
||||
pub preview_pool: Option<Arc<rayon::ThreadPool>>,
|
||||
pub desktop_enabled: bool,
|
||||
pub secret: String,
|
||||
pub key: Option<String>,
|
||||
|
|
@ -223,10 +225,14 @@ pub async fn get_thumbnail(
|
|||
.map_err(ErrorInternalServerError)?;
|
||||
if let Address::Hash(address_hash) = address {
|
||||
let preview_store = preview_store.clone();
|
||||
let preview_result = web::block(move || {
|
||||
preview_store.get(address_hash, query.mime, state.job_container.clone())
|
||||
})
|
||||
.await?;
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let _job_container = state.job_container.clone();
|
||||
state.preview_pool.as_ref().unwrap().spawn(move || {
|
||||
let result = preview_store.get(address_hash, query.mime, _job_container);
|
||||
tx.send(result).unwrap();
|
||||
});
|
||||
|
||||
let preview_result = rx.await.unwrap().map_err(ErrorInternalServerError)?;
|
||||
|
||||
if let Some(preview_path) = preview_result {
|
||||
let mut file = NamedFile::open(&preview_path)?.disable_content_disposition();
|
||||
|
|
|
|||
Loading…
Reference in New Issue