feat: add addressing/hashing of remote urls

cli/src/routes.rs

@@ -1,10 +1,11 @@
-use crate::extractors::{self};
+use crate::extractors;
 use crate::previews::PreviewStore;
 use crate::util::exec::block_background;
 use actix_files::NamedFile;
 use actix_multipart::Multipart;
 use actix_web::error::{
     ErrorBadRequest, ErrorInternalServerError, ErrorNotFound, ErrorUnauthorized,
+    ErrorUnprocessableEntity,
 };
 use actix_web::{delete, error, get, post, put, web, Either, Error, HttpResponse};
 use actix_web::{http, Responder};
@@ -25,7 +26,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
 use tempfile::NamedTempFile;
 use tracing::{debug, info, trace};
 use upend::addressing::{Address, Addressable};
-use upend::common::build;
+use upend::common::{build, APP_USER_AGENT};
 use upend::config::UpEndConfig;
 use upend::database::constants::{ADDED_ATTR, LABEL_ATTR};
 use upend::database::entry::{Entry, EntryValue, InvariantEntry};
@@ -33,7 +34,7 @@ use upend::database::hierarchies::{list_roots, resolve_path, UHierPath};
 use upend::database::lang::Query;
 use upend::database::stores::{Blob, UpStore};
 use upend::database::UpEndDatabase;
-use upend::util::hash::{b58_decode, b58_encode};
+use upend::util::hash::{b58_decode, b58_encode, hash};
 use upend::util::jobs;
 use url::Url;
 use uuid::Uuid;
@@ -643,32 +644,70 @@ pub async fn delete_object(
 //     Ok(HttpResponse::Ok().finish())
 // }
 
-#[derive(Deserialize)]
-pub struct GetAddressRequest {
-    attribute: Option<String>,
-    // url: Option<String>,
-}
-
 #[get("/api/address")]
 pub async fn get_address(
-    web::Query(query): web::Query<GetAddressRequest>,
+    web::Query(query): web::Query<HashMap<String, String>>,
 ) -> Result<HttpResponse, Error> {
-    let address = match query {
-        GetAddressRequest {
-            attribute: Some(attribute),
-        } => Address::Attribute(attribute),
-        _ => Err(ErrorBadRequest("Specify one of: `attribute`"))?,
+    let (address, immutable) = if let Some(attribute) = query.get("attribute") {
+        (Address::Attribute(attribute.into()), true)
+    } else if let Some(url) = query.get("url") {
+        (
+            Address::Url(Url::parse(url).map_err(ErrorBadRequest)?),
+            true,
+        )
+    } else if let Some(url) = query.get("url_content") {
+        let url = Url::parse(url).map_err(ErrorBadRequest)?;
+
+        const MAX_SIZE: usize = 128_000;
+
+        let client = reqwest::blocking::Client::builder()
+            .user_agent(APP_USER_AGENT.as_str())
+            .build()
+            .map_err(ErrorInternalServerError)?;
+
+        let response = client
+            .get(url)
+            .send()
+            .map_err(ErrorInternalServerError)?
+            .error_for_status()
+            .map_err(ErrorInternalServerError)?;
+
+        if let Some(content_length) = response.headers().get(reqwest::header::CONTENT_LENGTH) {
+            if let Some(content_length) = content_length
+                .to_str()
+                .ok()
+                .and_then(|cl| cl.parse::<usize>().ok())
+            {
+                if content_length > MAX_SIZE {
+                    return Err(ErrorBadRequest("Error: The response is too large."));
+                }
+            }
+        } else {
+            return Err(ErrorUnprocessableEntity(
+                "Error: Could not ascertain response size.",
+            ));
+        }
+
+        let bytes = response.bytes().map_err(ErrorInternalServerError)?;
+        let hash_result = hash(&bytes);
+        (Address::Hash(hash_result), false)
+    } else {
+        return Err(ErrorBadRequest(anyhow!(
+            "Specify one of: `attribute`, `url`, `url_content`."
+        )));
     };
 
-    Ok(HttpResponse::Ok()
-        .set_header(
+    let mut response = HttpResponse::Ok();
+    if immutable {
+        response.set_header(
             http::header::CACHE_CONTROL,
             CacheControl(vec![
                 CacheDirective::MaxAge(2678400),
                 CacheDirective::Extension("immutable".into(), None),
             ]),
-        )
-        .json(format!("{}", address)))
+        );
+    }
+    Ok(response.json(format!("{}", address)))
 }
 
 #[get("/api/all/attributes")]