refactor(cli): Blob is no longer tied just to filepaths
(this lays the foundation for future backing by e.g. S3, also in-memory blobs)
This commit is contained in:
parent
1efd45806a
commit
44d76aa8d4
8 changed files with 108 additions and 38 deletions
|
@ -9,7 +9,6 @@ use upend_base::{
|
||||||
constants::{ATTR_IN, ATTR_KEY, ATTR_LABEL, ATTR_OF},
|
constants::{ATTR_IN, ATTR_KEY, ATTR_LABEL, ATTR_OF},
|
||||||
entry::{Entry, EntryValue, InvariantEntry},
|
entry::{Entry, EntryValue, InvariantEntry},
|
||||||
};
|
};
|
||||||
use upend_db::stores::Blob;
|
|
||||||
use upend_db::{
|
use upend_db::{
|
||||||
jobs::{JobContainer, JobState},
|
jobs::{JobContainer, JobState},
|
||||||
stores::{fs::FILE_MIME_KEY, UpStore},
|
stores::{fs::FILE_MIME_KEY, UpStore},
|
||||||
|
@ -46,7 +45,7 @@ impl Extractor for ID3Extractor {
|
||||||
let files = store.retrieve(hash)?;
|
let files = store.retrieve(hash)?;
|
||||||
|
|
||||||
if let Some(file) = files.first() {
|
if let Some(file) = files.first() {
|
||||||
let file_path = file.get_file_path();
|
let file_path = file.get_file_path()?;
|
||||||
let mut job_handle = job_container.add_job(
|
let mut job_handle = job_container.add_job(
|
||||||
None,
|
None,
|
||||||
&format!(
|
&format!(
|
||||||
|
@ -98,7 +97,7 @@ impl Extractor for ID3Extractor {
|
||||||
file.write_all(&picture.data)?;
|
file.write_all(&picture.data)?;
|
||||||
let hash = store.store(
|
let hash = store.store(
|
||||||
connection,
|
connection,
|
||||||
Blob::from_filepath(&tmp_path),
|
tmp_path.into(),
|
||||||
None,
|
None,
|
||||||
Some(BlobMode::StoreOnly),
|
Some(BlobMode::StoreOnly),
|
||||||
context.clone(),
|
context.clone(),
|
||||||
|
@ -182,10 +181,6 @@ impl Extractor for ID3Extractor {
|
||||||
.query(format!("(matches @{} (contains \"ID3\") ?)", address).parse()?)?
|
.query(format!("(matches @{} (contains \"ID3\") ?)", address).parse()?)?
|
||||||
.is_empty();
|
.is_empty();
|
||||||
|
|
||||||
if is_extracted {
|
return Ok(!is_extracted);
|
||||||
return Ok(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(true)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -48,7 +48,7 @@ impl Extractor for ExifExtractor {
|
||||||
let files = store.retrieve(hash)?;
|
let files = store.retrieve(hash)?;
|
||||||
|
|
||||||
if let Some(file) = files.first() {
|
if let Some(file) = files.first() {
|
||||||
let file_path = file.get_file_path();
|
let file_path = file.get_file_path()?;
|
||||||
let mut job_handle = job_container.add_job(
|
let mut job_handle = job_container.add_job(
|
||||||
None,
|
None,
|
||||||
&format!(
|
&format!(
|
||||||
|
|
|
@ -55,7 +55,7 @@ impl Extractor for MediaExtractor {
|
||||||
let files = store.retrieve(hash)?;
|
let files = store.retrieve(hash)?;
|
||||||
|
|
||||||
if let Some(file) = files.first() {
|
if let Some(file) = files.first() {
|
||||||
let file_path = file.get_file_path();
|
let file_path = file.get_file_path()?;
|
||||||
let mut job_handle = job_container.add_job(
|
let mut job_handle = job_container.add_job(
|
||||||
None,
|
None,
|
||||||
&format!(
|
&format!(
|
||||||
|
|
|
@ -91,7 +91,7 @@ impl PreviewStore {
|
||||||
trace!("Calculating preview for {hash:?}...");
|
trace!("Calculating preview for {hash:?}...");
|
||||||
let files = self.store.retrieve(&hash)?;
|
let files = self.store.retrieve(&hash)?;
|
||||||
if let Some(file) = files.first() {
|
if let Some(file) = files.first() {
|
||||||
let file_path = file.get_file_path();
|
let file_path = file.get_file_path()?;
|
||||||
let mut job_handle = job_container.add_job(
|
let mut job_handle = job_container.add_job(
|
||||||
None,
|
None,
|
||||||
&format!("Creating preview for {:?}", file_path.file_name().unwrap()),
|
&format!("Creating preview for {:?}", file_path.file_name().unwrap()),
|
||||||
|
@ -102,21 +102,21 @@ impl PreviewStore {
|
||||||
let mime_type: Option<String> = if mime_type.is_some() {
|
let mime_type: Option<String> = if mime_type.is_some() {
|
||||||
mime_type
|
mime_type
|
||||||
} else {
|
} else {
|
||||||
tree_magic_mini::from_filepath(file_path).map(|m| m.into())
|
tree_magic_mini::from_filepath(&file_path).map(|m| m.into())
|
||||||
};
|
};
|
||||||
|
|
||||||
let preview = match mime_type {
|
let preview = match mime_type {
|
||||||
Some(tm) if tm.starts_with("text") => {
|
Some(tm) if tm.starts_with("text") => {
|
||||||
TextPath(file_path).get_thumbnail(options)
|
TextPath(&file_path).get_thumbnail(options)
|
||||||
}
|
}
|
||||||
Some(tm) if tm.starts_with("video") || tm == "application/x-matroska" => {
|
Some(tm) if tm.starts_with("video") || tm == "application/x-matroska" => {
|
||||||
VideoPath(file_path).get_thumbnail(options)
|
VideoPath(&file_path).get_thumbnail(options)
|
||||||
}
|
}
|
||||||
Some(tm) if tm.starts_with("audio") || tm == "application/x-riff" => {
|
Some(tm) if tm.starts_with("audio") || tm == "application/x-riff" => {
|
||||||
AudioPath(file_path).get_thumbnail(options)
|
AudioPath(&file_path).get_thumbnail(options)
|
||||||
}
|
}
|
||||||
Some(tm) if tm.starts_with("image") => {
|
Some(tm) if tm.starts_with("image") => {
|
||||||
ImagePath(file_path).get_thumbnail(options)
|
ImagePath(&file_path).get_thumbnail(options)
|
||||||
}
|
}
|
||||||
Some(unknown) => Err(anyhow!("No capability for {:?} thumbnails.", unknown)),
|
Some(unknown) => Err(anyhow!("No capability for {:?} thumbnails.", unknown)),
|
||||||
_ => Err(anyhow!("Unknown file type, or file doesn't exist.")),
|
_ => Err(anyhow!("Unknown file type, or file doesn't exist.")),
|
||||||
|
|
|
@ -38,8 +38,8 @@ use upend_base::hash::{b58_decode, b58_encode, sha256hash};
|
||||||
use upend_base::lang::Query;
|
use upend_base::lang::Query;
|
||||||
use upend_db::hierarchies::{list_roots, resolve_path, UHierPath};
|
use upend_db::hierarchies::{list_roots, resolve_path, UHierPath};
|
||||||
use upend_db::jobs;
|
use upend_db::jobs;
|
||||||
|
use upend_db::stores::UpStore;
|
||||||
use upend_db::stores::UpdateOptions;
|
use upend_db::stores::UpdateOptions;
|
||||||
use upend_db::stores::{Blob, UpStore};
|
|
||||||
use upend_db::BlobMode;
|
use upend_db::BlobMode;
|
||||||
use upend_db::OperationContext;
|
use upend_db::OperationContext;
|
||||||
use upend_db::UpEndDatabase;
|
use upend_db::UpEndDatabase;
|
||||||
|
@ -225,7 +225,7 @@ pub async fn get_raw(
|
||||||
.await?
|
.await?
|
||||||
.map_err(ErrorInternalServerError)?;
|
.map_err(ErrorInternalServerError)?;
|
||||||
if let Some(blob) = blobs.first() {
|
if let Some(blob) = blobs.first() {
|
||||||
let file_path = blob.get_file_path();
|
let file_path = blob.get_file_path().map_err(ErrorInternalServerError)?;
|
||||||
|
|
||||||
if query.native.is_none() {
|
if query.native.is_none() {
|
||||||
return Ok(Either::Left(
|
return Ok(Either::Left(
|
||||||
|
@ -253,7 +253,7 @@ pub async fn get_raw(
|
||||||
info!("Opening {:?}...", file_path);
|
info!("Opening {:?}...", file_path);
|
||||||
let mut response = HttpResponse::NoContent();
|
let mut response = HttpResponse::NoContent();
|
||||||
let path = if !file_path.is_executable() || state.config.trust_executables {
|
let path = if !file_path.is_executable() || state.config.trust_executables {
|
||||||
file_path
|
&file_path
|
||||||
} else {
|
} else {
|
||||||
response
|
response
|
||||||
.append_header((
|
.append_header((
|
||||||
|
@ -317,10 +317,10 @@ pub async fn head_raw(
|
||||||
.await?
|
.await?
|
||||||
.map_err(ErrorInternalServerError)?;
|
.map_err(ErrorInternalServerError)?;
|
||||||
if let Some(blob) = blobs.first() {
|
if let Some(blob) = blobs.first() {
|
||||||
let file_path = blob.get_file_path();
|
let file_path = blob.get_file_path().map_err(ErrorInternalServerError)?;
|
||||||
|
|
||||||
let mut response = HttpResponse::NoContent();
|
let mut response = HttpResponse::NoContent();
|
||||||
if let Some(mime_type) = tree_magic_mini::from_filepath(file_path) {
|
if let Some(mime_type) = tree_magic_mini::from_filepath(&file_path) {
|
||||||
if let Ok(mime) = mime_type.parse::<mime::Mime>() {
|
if let Ok(mime) = mime_type.parse::<mime::Mime>() {
|
||||||
return Ok(response.content_type(mime).finish());
|
return Ok(response.content_type(mime).finish());
|
||||||
}
|
}
|
||||||
|
@ -672,7 +672,7 @@ pub async fn put_blob(
|
||||||
_store
|
_store
|
||||||
.store(
|
.store(
|
||||||
&connection,
|
&connection,
|
||||||
Blob::from_filepath(file.path()),
|
file.path().into(),
|
||||||
_filename,
|
_filename,
|
||||||
options.blob_mode,
|
options.blob_mode,
|
||||||
OperationContext {
|
OperationContext {
|
||||||
|
|
|
@ -3,7 +3,7 @@ use self::db::files;
|
||||||
use super::{Blob, StoreError, UpStore, UpdateOptions, UpdatePathOutcome};
|
use super::{Blob, StoreError, UpStore, UpdateOptions, UpdatePathOutcome};
|
||||||
use crate::hierarchies::{resolve_path, resolve_path_cached, ResolveCache, UHierPath, UNode};
|
use crate::hierarchies::{resolve_path, resolve_path_cached, ResolveCache, UHierPath, UNode};
|
||||||
use crate::jobs::{JobContainer, JobHandle};
|
use crate::jobs::{JobContainer, JobHandle};
|
||||||
use crate::util::hash_at_path;
|
use crate::util::{hash, hash_at_path};
|
||||||
use crate::{
|
use crate::{
|
||||||
BlobMode, ConnectionOptions, LoggingHandler, OperationContext, UpEndConnection, UpEndDatabase,
|
BlobMode, ConnectionOptions, LoggingHandler, OperationContext, UpEndConnection, UpEndDatabase,
|
||||||
UPEND_SUBDIR,
|
UPEND_SUBDIR,
|
||||||
|
@ -628,15 +628,13 @@ impl FsStore {
|
||||||
|
|
||||||
impl From<db::OutFile> for Blob {
|
impl From<db::OutFile> for Blob {
|
||||||
fn from(of: db::OutFile) -> Self {
|
fn from(of: db::OutFile) -> Self {
|
||||||
Blob { file_path: of.path }
|
of.path.into()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<db::File> for Blob {
|
impl From<db::File> for Blob {
|
||||||
fn from(f: db::File) -> Self {
|
fn from(f: db::File) -> Self {
|
||||||
Blob {
|
PathBuf::from(f.path).into()
|
||||||
file_path: PathBuf::from(f.path),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -667,8 +665,11 @@ impl UpStore for FsStore {
|
||||||
blob_mode: Option<BlobMode>,
|
blob_mode: Option<BlobMode>,
|
||||||
context: OperationContext,
|
context: OperationContext,
|
||||||
) -> Result<UpMultihash, super::StoreError> {
|
) -> Result<UpMultihash, super::StoreError> {
|
||||||
let file_path = blob.get_file_path();
|
let hash = hash(
|
||||||
let hash = hash_at_path(file_path).map_err(|e| StoreError::Unknown(e.to_string()))?;
|
blob.read()
|
||||||
|
.map_err(|e| StoreError::Unknown(e.to_string()))?,
|
||||||
|
)
|
||||||
|
.map_err(|e| StoreError::Unknown(e.to_string()))?;
|
||||||
|
|
||||||
let existing_files = self.retrieve(&hash)?;
|
let existing_files = self.retrieve(&hash)?;
|
||||||
|
|
||||||
|
@ -687,7 +688,8 @@ impl UpStore for FsStore {
|
||||||
};
|
};
|
||||||
|
|
||||||
let final_path = self.path.join(final_name);
|
let final_path = self.path.join(final_name);
|
||||||
fs::copy(file_path, &final_path).map_err(|e| StoreError::Unknown(e.to_string()))?;
|
blob.copy_to(&final_path)
|
||||||
|
.map_err(|e| StoreError::Unknown(e.to_string()))?;
|
||||||
|
|
||||||
let upath = if let Some(bm) = blob_mode {
|
let upath = if let Some(bm) = blob_mode {
|
||||||
self.path_to_upath(&final_path, bm)
|
self.path_to_upath(&final_path, bm)
|
||||||
|
@ -1112,7 +1114,7 @@ mod test {
|
||||||
paths.iter().for_each(|path| {
|
paths.iter().for_each(|path| {
|
||||||
let upath: UHierPath = path.parse().unwrap();
|
let upath: UHierPath = path.parse().unwrap();
|
||||||
assert!(
|
assert!(
|
||||||
resolve_path(&connection, &upath, false, OperationContext::default()).is_ok(),
|
resolve_path(connection, &upath, false, OperationContext::default()).is_ok(),
|
||||||
"Failed: {}",
|
"Failed: {}",
|
||||||
upath
|
upath
|
||||||
);
|
);
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
use std::io::Read;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use super::{UpEndConnection, UpEndDatabase};
|
use super::{UpEndConnection, UpEndDatabase};
|
||||||
|
@ -28,19 +29,86 @@ impl std::error::Error for StoreError {}
|
||||||
|
|
||||||
type Result<T> = std::result::Result<T, StoreError>;
|
type Result<T> = std::result::Result<T, StoreError>;
|
||||||
|
|
||||||
pub struct Blob {
|
pub enum Blob {
|
||||||
file_path: PathBuf,
|
File(PathBuf),
|
||||||
|
Data(Vec<u8>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::io::Read for Blob {
|
||||||
|
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||||
|
match self {
|
||||||
|
Blob::File(path) => {
|
||||||
|
let mut file = std::fs::File::open(path)?;
|
||||||
|
file.read(buf)
|
||||||
|
}
|
||||||
|
Blob::Data(data) => {
|
||||||
|
let len = std::cmp::min(data.len(), buf.len());
|
||||||
|
buf[..len].copy_from_slice(&data[..len]);
|
||||||
|
Ok(len)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Blob {
|
impl Blob {
|
||||||
pub fn from_filepath<P: AsRef<Path>>(path: P) -> Blob {
|
pub fn read(&self) -> Result<Vec<u8>> {
|
||||||
Blob {
|
match self {
|
||||||
file_path: PathBuf::from(path.as_ref()),
|
Blob::File(path) => {
|
||||||
|
let file = std::fs::File::open(path)
|
||||||
|
.map_err(|err| StoreError::Unknown(err.to_string()))?;
|
||||||
|
let mut reader = std::io::BufReader::new(file);
|
||||||
|
let mut buffer = Vec::new();
|
||||||
|
reader
|
||||||
|
.read_to_end(&mut buffer)
|
||||||
|
.map_err(|err| StoreError::Unknown(err.to_string()))?;
|
||||||
|
Ok(buffer)
|
||||||
|
}
|
||||||
|
Blob::Data(data) => Ok(data.clone()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_file_path(&self) -> &Path {
|
pub fn copy_to(&self, path: &Path) -> Result<()> {
|
||||||
self.file_path.as_path()
|
match self {
|
||||||
|
Blob::File(src) => {
|
||||||
|
std::fs::copy(src, path).map_err(|err| StoreError::Unknown(err.to_string()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Blob::Data(data) => {
|
||||||
|
std::fs::write(path, data).map_err(|err| StoreError::Unknown(err.to_string()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_file_path(&self) -> Result<PathBuf> {
|
||||||
|
match self {
|
||||||
|
Blob::File(path) => Ok(path.clone()),
|
||||||
|
Blob::Data(_) => Err(StoreError::Unknown("Blob is not a file".to_string())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&Path> for Blob {
|
||||||
|
fn from(path: &Path) -> Blob {
|
||||||
|
Blob::File(path.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<PathBuf> for Blob {
|
||||||
|
fn from(path: PathBuf) -> Blob {
|
||||||
|
Blob::File(path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<Vec<u8>> for Blob {
|
||||||
|
fn from(data: Vec<u8>) -> Blob {
|
||||||
|
Blob::Data(data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<String> for Blob {
|
||||||
|
fn from(data: String) -> Blob {
|
||||||
|
Blob::Data(data.into_bytes())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -34,6 +34,11 @@ impl std::io::Write for LoggerSink {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn hash<T: AsRef<[u8]>>(input: T) -> anyhow::Result<UpMultihash> {
|
||||||
|
let hash = upend_base::hash::sha256hash(input)?;
|
||||||
|
Ok(hash)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn hash_at_path<P: AsRef<Path>>(path: P) -> anyhow::Result<UpMultihash> {
|
pub fn hash_at_path<P: AsRef<Path>>(path: P) -> anyhow::Result<UpMultihash> {
|
||||||
let path = path.as_ref();
|
let path = path.as_ref();
|
||||||
trace!("Hashing {:?}...", path);
|
trace!("Hashing {:?}...", path);
|
||||||
|
|
Loading…
Reference in a new issue