thm
/
upend
1
0
Fork 0
Code Issues 73 Pull Requests Packages Projects Releases 5 Wiki Activity

add s-exp query language parsing 

also rename t/k/v to e/a/v, misc. fixes
feat/vaults
Tomáš Mládek 2021-02-07 20:18:55 +01:00
parent bb2a8e909f
commit f950e02113
7 changed files with 1100 additions and 866 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1694
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

5
Cargo.toml
View File

@ -15,8 +15,11 @@ log = "0.4"
anyhow = "1.0"
thiserror = "1.0"
rayon = "1.4.0"
diesel = { version = "1.4", features = ["sqlite", "r2d2", "chrono", "serde_json"] }
diesel_migrations = "1.4"
libsqlite3-sys = { version = "^0", features = ["bundled"] }
actix = "0.9.0"
actix-files = "0.2.2"
@ -27,7 +30,7 @@ actix_derive = "0.3.2"
chrono = { version = "0.4", features = ["serde"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
rayon = "1.4.0"
lexpr = "0.2.6"
bs58 = "0.3.1"
filebuffer = "0.4.0"

9
src/addressing.rs
View File

@ -9,6 +9,7 @@ use uuid::Uuid;
use crate::hash::{encode, Hash};
use serde::export::Formatter;
use thiserror::private::DisplayAsDisplay;
use std::str::FromStr;
#[derive(Clone, PartialEq)]
pub enum Address {
@ -57,6 +58,14 @@ impl Address {
}
}
impl FromStr for Address {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Address::decode(s.as_ref())
}
}
impl std::fmt::Display for Address {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", encode(self.encode().map_err(|_| std::fmt::Error)?))

209
src/database.rs
View File

@ -7,13 +7,17 @@ use diesel::debug_query;
use diesel::prelude::*;
use diesel::r2d2::{self, ConnectionManager};
use diesel::sqlite::{Sqlite, SqliteConnection};
use lexpr::value::Value::Symbol;
use lexpr::Value::Cons;
use log::{debug, trace};
use serde::export::Formatter;
use serde_json::{json, Value};
use serde_json::json;
use std::borrow::Borrow;
use std::convert::TryFrom;
use std::fs;
use std::io::{Cursor, Write};
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::time::Duration;
#[derive(Debug, Clone)]
@ -246,19 +250,194 @@ pub fn remove_object<C: Connection<Backend = Sqlite>>(
Ok(diesel::delete(matches).execute(connection)?)
}
pub enum QueryComponent<T> {
#[derive(Debug)]
pub enum QueryComponent<T>
where
T: FromStr,
{
Exact(T),
ILike(T),
In(Vec<T>),
Any,
}
// #[derive(Debug)]
// pub enum StringOrLikeString {
// String(String),
// Like(String)
// }
#[derive(Debug)]
pub struct EntryQuery {
pub target: QueryComponent<Address>,
pub key: QueryComponent<String>,
pub entity: QueryComponent<Address>,
pub attribute: QueryComponent<String>,
pub value: QueryComponent<EntryValue>,
}
#[derive(Debug)]
pub enum QueryPart {
Matches(EntryQuery),
Type(String),
}
#[derive(Debug)]
pub enum QueryQualifier {
AND,
OR,
}
#[derive(Debug)]
pub struct MultiQuery {
pub qualifier: QueryQualifier,
pub queries: Vec<Box<Query>>,
}
#[derive(Debug)]
pub enum Query {
SingleQuery(QueryPart),
MultiQuery(MultiQuery),
}
impl Query {
pub fn from_sexp(expression: &lexpr::Value) -> Result<Self> {
if let Cons(value) = expression {
if let Symbol(symbol) = value.car() {
match symbol.borrow() {
"matches" => {
let (cons_vec, _) = value.clone().into_vec();
if let [_, entity, attribute, value] = &cons_vec[..] {
let entity = Query::parse_component::<Address>(entity)?;
let attribute = Query::parse_component::<String>(attribute)?;
let value = Query::parse_component::<EntryValue>(value)?;
Ok(Query::SingleQuery(QueryPart::Matches(EntryQuery {
entity,
attribute,
value,
})))
} else {
Err(anyhow!(
"Malformed expression: Wrong number of arguments to 'matches'."
))
}
}
"type" => {
let (cons_vec, _) = value.clone().into_vec();
if let [_, type_name] = &cons_vec[..] {
if let lexpr::Value::String(type_name_str) = type_name {
Ok(Query::SingleQuery(QueryPart::Type(
type_name_str.to_string(),
)))
} else {
Err(anyhow!(
"Malformed expression: Type must be specified as a string."
))
}
} else {
Err(anyhow!(
"Malformed expression: Wrong number of arguments to 'type'."
))
}
}
"and" | "or" => {
let (cons_vec, _) = value.clone().into_vec();
if let Some(split) = cons_vec.split_first() {
let sub_expressions = split.1;
let values: Result<Vec<Query>> = sub_expressions
.into_iter()
.map(|value| Query::from_sexp(value))
.collect();
Ok(Query::MultiQuery(MultiQuery {
qualifier: match symbol.borrow() {
"and" => QueryQualifier::AND,
_ => QueryQualifier::OR,
},
queries: values?.into_iter().map(|sq| Box::new(sq)).collect(),
}))
} else {
Err(anyhow!(
"Malformed expression: sub-query list cannot be empty.",
))
}
}
_ => Err(anyhow!(format!(
"Malformed expression: Unknown symbol '{}'.",
symbol
))),
}
} else {
Err(anyhow!(format!(
"Malformed expression: Value '{:?}' is not a symbol.",
value
)))
}
} else {
Err(anyhow!("Malformed expression: Not a list."))
}
}
fn parse_component<T: FromStr>(value: &lexpr::Value) -> Result<QueryComponent<T>> {
match value {
Cons(cons) => {
if let Symbol(symbol) = cons.car() {
match symbol.borrow() {
"in" => {
let (cons_vec, _) = cons.clone().into_vec();
if let Some(split) = cons_vec.split_first() {
let args = split.1;
let values: Result<Vec<T>, _> = args.into_iter().map(|value| {
if let lexpr::Value::String(str) = value {
if let Ok(value) = T::from_str(str.borrow()) {
Ok(value)
} else {
Err(anyhow!(format!("Malformed expression: Conversion of inner value '{}' from string failed.", str)))
}
} else {
Err(anyhow!("Malformed expression: Inner value list must be comprised of strings."))
}
}).collect();
Ok(QueryComponent::In(values?))
} else {
Err(anyhow!(
"Malformed expression: Inner value cannot be empty."
))
}
}
_ => Err(anyhow!(format!(
"Malformed expression: Unknown symbol {}",
symbol
))),
}
} else {
Err(anyhow!(format!(
"Malformed expression: Inner value '{:?}' is not a symbol.",
value
)))
}
}
lexpr::Value::String(str) => {
if let Ok(value) = T::from_str(str.borrow()) {
Ok(QueryComponent::Exact(value))
} else {
Err(anyhow!(format!(
"Malformed expression: Conversion of inner value '{}' from string failed.",
str
)))
}
}
lexpr::Value::Symbol(symbol) => match symbol.borrow() {
"?" => Ok(QueryComponent::Any),
_ => Err(anyhow!(format!(
"Malformed expression: Unknown symbol {}",
symbol
))),
},
_ => Err(anyhow!(
"Malformed expression: Inner value not a string, list or '?'."
)),
}
}
}
pub fn query_entries<C: Connection<Backend = Sqlite>>(
connection: &C,
entry_query: EntryQuery,
@ -267,20 +446,20 @@ pub fn query_entries<C: Connection<Backend = Sqlite>>(
let mut query = data.into_boxed();
query = match entry_query.target {
query = match entry_query.entity {
QueryComponent::Exact(q_target) => query.filter(target.eq(q_target.encode()?)),
QueryComponent::In(q_targets) => {
let targets: Result<Vec<_>, _> = q_targets.into_iter().map(|t| t.encode()).collect();
query.filter(target.eq_any(targets?))
}
QueryComponent::ILike(_) => return Err(anyhow!("Cannot query Address alike.")),
// QueryComponent::ILike(_) => return Err(anyhow!("Cannot query Address alike.")),
QueryComponent::Any => query,
};
query = match entry_query.key {
query = match entry_query.attribute {
QueryComponent::Exact(q_key) => query.filter(key.eq(q_key)),
QueryComponent::In(q_keys) => query.filter(key.eq_any(q_keys)),
QueryComponent::ILike(q_key) => query.filter(key.like(format!("%{}%", q_key))),
// QueryComponent::ILike(q_key) => query.filter(key.like(format!("%{}%", q_key))),
QueryComponent::Any => query,
};
@ -290,12 +469,12 @@ pub fn query_entries<C: Connection<Backend = Sqlite>>(
let values: Result<Vec<_>, _> = q_values.into_iter().map(|v| v.to_str()).collect();
query.filter(value.eq_any(values?))
}
QueryComponent::ILike(EntryValue::Value(Value::String(q_value_string))) => {
query.filter(value.like(format!("%{}%", q_value_string)))
}
QueryComponent::ILike(_) => {
return Err(anyhow!("Only string Values can be queried alike."))
}
// QueryComponent::ILike(EntryValue::Value(Value::String(q_value_string))) => {
// query.filter(value.like(format!("%{}%", q_value_string)))
// }
// QueryComponent::ILike(_) => {
// return Err(anyhow!("Only string Values can be queried alike."));
// }
QueryComponent::Any => query,
};

28
src/filesystem.rs
View File

@ -134,8 +134,8 @@ pub fn list_roots<C: Connection<Backend = Sqlite>>(connection: &C) -> Result<Vec
let all_directories: Vec<Entry> = query_entries(
connection,
EntryQuery {
target: QueryComponent::Any,
key: QueryComponent::Exact(DIR_KEY.to_string()),
entity: QueryComponent::Any,
attribute: QueryComponent::Exact(DIR_KEY.to_string()),
value: QueryComponent::Any,
},
)?;
@ -143,8 +143,8 @@ pub fn list_roots<C: Connection<Backend = Sqlite>>(connection: &C) -> Result<Vec
let directories_with_parents: Vec<Address> = query_entries(
connection,
EntryQuery {
target: QueryComponent::Any,
key: QueryComponent::Exact(DIR_HAS_KEY.to_string()),
entity: QueryComponent::Any,
attribute: QueryComponent::Exact(DIR_HAS_KEY.to_string()),
value: QueryComponent::Any,
},
)?
@ -172,8 +172,8 @@ pub async fn list_directory<C: Connection<Backend = Sqlite>>(
query_entries(
connection,
EntryQuery {
target: QueryComponent::Exact(last.clone()),
key: QueryComponent::Exact(DIR_HAS_KEY.to_string()),
entity: QueryComponent::Exact(last.clone()),
attribute: QueryComponent::Exact(DIR_HAS_KEY.to_string()),
value: QueryComponent::Any,
},
)?
@ -202,8 +202,8 @@ pub fn fetch_or_create_dir<C: Connection<Backend = Sqlite>>(
let directories: Vec<Address> = query_entries(
connection,
EntryQuery {
target: QueryComponent::Any,
key: QueryComponent::Exact(String::from(DIR_KEY)),
entity: QueryComponent::Any,
attribute: QueryComponent::Exact(String::from(DIR_KEY)),
value: QueryComponent::Exact(dir_value.clone()),
},
)?
@ -216,8 +216,8 @@ pub fn fetch_or_create_dir<C: Connection<Backend = Sqlite>>(
let parent_has: Vec<Address> = query_entries(
connection,
EntryQuery {
target: QueryComponent::Exact(address),
key: QueryComponent::Exact(String::from(DIR_HAS_KEY)),
entity: QueryComponent::Exact(address),
attribute: QueryComponent::Exact(String::from(DIR_HAS_KEY)),
value: QueryComponent::Any,
},
)?
@ -289,7 +289,7 @@ pub async fn reimport_directory(pool: DbPool, directory: PathBuf) {
let result = actix_web::web::block(move || _reimport_directory(pool, directory)).await;
if result.is_err() {
let err = result.err().unwrap();
error!("Update did not succeed! {}", err);
error!("Update did not succeed! {:?}", err);
}
}
@ -471,9 +471,9 @@ pub fn lookup_by_filename<C: Connection<Backend = Sqlite>>(
let entity_addresses = query_entries(
connection,
EntryQuery {
target: QueryComponent::Any,
key: QueryComponent::In(vec![DIR_KEY.to_string(), FILENAME_KEY.to_string()]), // ?
value: QueryComponent::ILike(dir_value),
entity: QueryComponent::Any,
attribute: QueryComponent::In(vec![DIR_KEY.to_string(), FILENAME_KEY.to_string()]), // ?
value: QueryComponent::Exact(dir_value), // ??
},
)?;

1
src/main.rs
View File

@ -83,6 +83,7 @@ fn main() -> Result<()> {
App::new()
.data(state.clone())
.wrap(middleware::Logger::default())
.service(routes::get_query)
.service(routes::get_raw)
.service(routes::get_object)
.service(routes::list_hier)

20
src/routes.rs
View File

@ -1,5 +1,5 @@
use crate::addressing::Address;
use crate::database::{remove_object, retrieve_file, retrieve_object, DbPool, Entry};
use crate::database::{remove_object, retrieve_file, retrieve_object, DbPool, Entry, Query};
use crate::filesystem::{list_directory, lookup_by_filename, UPath};
use crate::hash::{decode, encode};
use actix_files::NamedFile;
@ -116,6 +116,24 @@ pub async fn get_lookup(
))
}
#[derive(Deserialize)]
pub struct QueryRequest {
query: String,
}
#[get("/api/query")]
pub async fn get_query(
state: web::Data<State>,
web::Query(info): web::Query<QueryRequest>,
) -> Result<HttpResponse, Error> {
let connection = state.db_pool.get().map_err(ErrorInternalServerError)?;
let result = lexpr::from_str(info.query.as_str()).map_err(ErrorInternalServerError)?;
let query = Query::from_sexp(&result).map_err(ErrorInternalServerError)?;
Ok(HttpResponse::Ok().json(format!("{:?}", query)))
}
#[post("/api/refresh")]
pub async fn api_refresh(state: web::Data<State>) -> Result<HttpResponse, Error> {
let _pool = state.db_pool.clone();