(cargo-release) start next development iteration 0.10.4-alpha.0

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "gardenserver"
-version = "0.5.6"
+version = "0.10.4-alpha.0"
 authors = ["Tomáš Mládek <t@mldk.cz>"]
 edition = "2018"
 publish = false
@@ -24,7 +24,13 @@ pulldown-cmark = "0.8.0"
 
 tera = "1"
 
+kuchiki = "0.8.1"
+html5ever = "*"
+markup5ever = "*"
+
 chrono = "0.4"
+timeago = "0.3.0"
 regex = "1"
 linkify = "0.5"
+slug = "0.1.4"
 percent-encoding = "2.1.0"

src/main.rs

@@ -1,24 +1,33 @@
+use crate::markup5ever::tendril::TendrilSink;
 use actix_files::NamedFile;
 use actix_web::error::ErrorInternalServerError;
 use actix_web::{error, get, http, middleware, web, App, Error, HttpResponse, HttpServer};
 use anyhow::anyhow;
 use chrono::{DateTime, Local};
 use clap::{App as ClapApp, Arg};
+use html5ever::serialize::{serialize, SerializeOpts};
+use kuchiki::{parse_fragment, Attribute, ExpandedName, NodeRef};
 use linkify::LinkFinder;
 use log::{info, trace};
+use markup5ever::QualName;
 use percent_encoding::{percent_decode_str, utf8_percent_encode};
 use pulldown_cmark::{html, Event, Options, Parser, Tag};
 use regex::{Captures, Regex};
+use slug::slugify;
+use std::cmp::Reverse;
 use std::collections::HashMap;
 use std::fs::File;
 use std::io::Read;
 use std::net::SocketAddr;
 use std::path::{Path, PathBuf};
 use std::sync::Mutex;
-use std::time::SystemTime;
+use std::time::{Duration, SystemTime};
 use std::{env, fs};
 use tera::{Context, Tera};
 
+#[macro_use]
+extern crate markup5ever;
+
 #[derive(Clone)]
 struct State {
     garden_dir: PathBuf,
@@ -36,6 +45,7 @@ struct MutableState {
 struct GardenCache {
     pages: HashMap<String, ParsedPage>,
     files: Vec<PathBuf>,
+    tags: HashMap<String, u32>,
 }
 
 impl Default for GardenCache {
@@ -43,6 +53,7 @@ impl Default for GardenCache {
         GardenCache {
             pages: HashMap::new(),
             files: vec![],
+            tags: HashMap::new(),
         }
     }
 }
@@ -113,7 +124,7 @@ fn main() -> anyhow::Result<()> {
         .unwrap()
         .parse()
         .expect("Incorrect bind format.");
-    info!("Starting server at: {}", &bind);
+    info!("Starting server at: http://{}", &bind);
 
     let mutable_state = web::Data::new(MutableState {
         garden_cache: Mutex::new(update_garden(directory, GardenCache::default())?),
@@ -186,10 +197,12 @@ async fn render(
             .finish());
     }
 
+    // If the path is not a markdown file (e.g. photos), just return it as it is.
     if full_path.exists() && !path.ends_with(".md") {
-        return Ok(NamedFile::open(full_path)?.into_response(&request)?);
+        return NamedFile::open(full_path)?.into_response(&request);
     }
 
+    // Otherwise, retrieve it and check backlinks
     let filename = full_path
         .components()
         .last()
@@ -211,8 +224,75 @@ async fn render(
         }
     }
 
-    let page = cache.pages.get(path.as_ref());
+    // Special case - graph view
+    let mut graph_page = ParsedPage {
+        timestamp: None,
+        title: "".to_string(),
+        html: "".to_string(),
+        links: vec![],
+    };
 
+    let page = if path.as_str() != "!graph" {
+        cache.pages.get(path.as_ref())
+    } else {
+        let mut context = Context::new();
+
+        let mut nodes: Vec<HashMap<String, String>> = vec![];
+        let mut links: Vec<HashMap<String, String>> = vec![];
+
+        let page_ids: Vec<String> = cache.pages.keys().map(|n| normalize_name(n)).collect();
+
+        cache.pages.iter().for_each(|(path, page)| {
+            let normalized_path = normalize_name(path);
+            nodes.push(
+                [("id".to_string(), normalized_path.clone())]
+                    .iter()
+                    .cloned()
+                    .collect(),
+            );
+            page.links
+                .iter()
+                .map(|l| normalize_name(l))
+                .filter(|link| page_ids.contains(link))
+                .for_each(|link| {
+                    links.push(
+                        [
+                            ("source".to_string(), normalized_path.clone()),
+                            ("target".to_string(), link),
+                        ]
+                        .iter()
+                        .cloned()
+                        .collect(),
+                    )
+                })
+        });
+
+        context.insert("nodes", &nodes);
+        context.insert("links", &links);
+
+        graph_page.title = "Graph View".to_string();
+        graph_page.html = data
+            .tera
+            .render("graph.html", &context)
+            .map_err(ErrorInternalServerError)?;
+        Some(&graph_page)
+    };
+
+    // Recently changed
+    let mut recently_changed = cache
+        .pages
+        .clone()
+        .into_iter()
+        .filter_map(|(path, page)| {
+            page.timestamp
+                .map(|ts| (path, SystemTime::now().duration_since(ts).unwrap()))
+        })
+        .collect::<Vec<(String, Duration)>>();
+    recently_changed.sort_by_key(|i| i.1);
+
+    let timeago = timeago::Formatter::new();
+
+    // Render context generation
     let mut context = Context::new();
     context.insert("version", VERSION);
     context.insert(
@@ -220,6 +300,19 @@ async fn render(
         data.title.as_ref().unwrap_or(&"Digital Garden".to_string()),
     );
     context.insert("files", &cache.files);
+
+    let mut tags: Vec<(&String, &u32)> = cache.tags.iter().collect();
+    tags.sort_by_key(|(t, _)| *t);
+    tags.sort_by_key(|(_, n)| Reverse(*n));
+    context.insert("tags", &tags);
+
+    context.insert(
+        "recently_changed",
+        &recently_changed
+            .into_iter()
+            .map(|(path, duration)| (path, timeago.convert(duration)))
+            .collect::<Vec<(String, String)>>(),
+    );
     context.insert(
         "page_title",
         &match page {
@@ -303,6 +396,7 @@ fn update_garden<P: AsRef<Path>>(
     }
 
     let mut pages = current.pages;
+    let mut tags = current.tags;
 
     let markdown_paths = files
         .iter()
@@ -323,7 +417,7 @@ fn update_garden<P: AsRef<Path>>(
         let mut file_string = String::new();
         file.read_to_string(&mut file_string)?;
         let markdown_source = preprocess_markdown(file_string);
-        let result = GardenParser::parse(&markdown_source);
+        let result = parse_garden(&markdown_source)?;
 
         pages.insert(
             String::from(path.to_str().unwrap()),
@@ -344,72 +438,70 @@ fn update_garden<P: AsRef<Path>>(
                 },
             },
         );
+
+        result.tags.into_iter().for_each(|tag| {
+            *tags.entry(tag).or_insert(0) += 1;
+        });
     }
 
-    let result = GardenCache { files, pages };
+    let result = GardenCache { pages, files, tags };
     trace!("{:#?}", result);
     Ok(result)
 }
 
-struct GardenParser<'a> {
-    parser: Parser<'a>,
-    last_nontext_event: Option<Event<'a>>,
-    current_top_heading: u32,
-    top_heading_text: &'a mut Option<String>,
-    links: &'a mut Vec<String>,
-}
-
 struct ParseResult {
     html: String,
     title: Option<String>,
     links: Vec<String>,
+    tags: Vec<String>,
 }
 
-impl<'a> GardenParser<'a> {
-    fn parse<S: AsRef<str>>(text: &'a S) -> ParseResult {
-        let mut title: Option<String> = None;
-        let mut links: Vec<String> = vec![];
+fn parse_garden<S: AsRef<str>>(text: S) -> anyhow::Result<ParseResult> {
+    let mut current_top_heading = 999;
+    let mut top_heading_text: Option<String> = None;
 
-        let parser = GardenParser {
-            parser: Parser::new_ext(text.as_ref(), Options::all()),
-            last_nontext_event: None,
-            current_top_heading: 999,
-            top_heading_text: &mut title,
-            links: &mut links,
-        };
+    let mut last_nontext_event: Option<Event> = None;
 
-        let mut html = String::new();
-        html::push_html(&mut html, parser);
+    let mut links: Vec<String> = vec![];
+    let mut tags: Vec<String> = vec![];
 
-        ParseResult { html, title, links }
-    }
-}
-
-impl<'a> Iterator for GardenParser<'a> {
-    type Item = Event<'a>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let event = self.parser.next();
-
-        if let Some(event) = &event {
-            if let Event::Start(Tag::Link(_, str, _)) = &event {
-                self.links.push(str.to_string());
-            }
-
-            if let Some(Event::Start(Tag::Heading(hl))) = self.last_nontext_event {
-                if hl < self.current_top_heading {
-                    self.current_top_heading = hl;
-                    if let Event::Text(str) = &event {
-                        *self.top_heading_text = Some(str.clone().into_string());
-                    }
-                }
-            }
-
-            self.last_nontext_event = Some(event.clone());
+    let parser = Parser::new_ext(text.as_ref(), Options::all()).map(|event| {
+        if let Event::Start(Tag::Link(_, dest, _)) = &event {
+            links.push(dest.to_string());
         }
 
+        if let Some(Event::Start(Tag::Link(_, _, _))) = &last_nontext_event {
+            if let Event::Text(str) = &event {
+                if str.starts_with('#') {
+                    tags.push(str[1..].to_string());
+                }
+            }
+        }
+
+        if let Some(Event::Start(Tag::Heading(hl))) = last_nontext_event {
+            if hl < current_top_heading {
+                current_top_heading = hl;
+                if let Event::Text(str) = &event {
+                    top_heading_text = Some(str.clone().into_string());
+                }
+            }
+        }
+
+        last_nontext_event = Some(event.clone());
+
         event
-    }
+    });
+
+    let mut html = String::new();
+    html::push_html(&mut html, parser);
+    html = postprocess_html(html)?;
+
+    Ok(ParseResult {
+        html,
+        title: top_heading_text,
+        links,
+        tags,
+    })
 }
 
 fn preprocess_markdown(string: String) -> String {
@@ -417,7 +509,7 @@ fn preprocess_markdown(string: String) -> String {
     let finder = LinkFinder::new();
 
     let result = double_brackets
-        .replace_all(string.as_str(), |caps: &Captures| {
+        .replace_all(&string, |caps: &Captures| {
             format!(
                 "[{}]({})",
                 &caps[1],
@@ -426,6 +518,17 @@ fn preprocess_markdown(string: String) -> String {
         })
         .to_string();
 
+    let tags = Regex::new(r"#([\w]+)").unwrap();
+    let result = tags
+        .replace_all(&result, |caps: &Captures| {
+            format!(
+                "[{}]({})",
+                &caps[0],
+                utf8_percent_encode(&caps[1], percent_encoding::NON_ALPHANUMERIC)
+            )
+        })
+        .to_string();
+
     let result_vec = Vec::from(result.as_str());
     let start_delims = vec![b'(', b'<'];
     let end_delims = vec![b')', b'>'];
@@ -449,8 +552,47 @@ fn preprocess_markdown(string: String) -> String {
     result_string
 }
 
+fn postprocess_html<T: AsRef<str>>(document: T) -> anyhow::Result<String> {
+    let document_bytes = String::from(document.as_ref());
+    let frag = parse_fragment(QualName::new(None, ns!(html), local_name!("body")), vec![])
+        .from_utf8()
+        .read_from(&mut document_bytes.as_bytes())
+        .unwrap();
+
+    frag.select("h1,h2,h3,h4,h5")
+        .unwrap()
+        .into_iter()
+        .for_each(|el| {
+            let id = slugify(el.text_contents());
+            el.attributes.borrow_mut().insert("id", id.clone());
+            el.as_node().prepend(NodeRef::new_element(
+                QualName::new(None, ns!(html), local_name!("a")),
+                vec![
+                    (
+                        ExpandedName::new(ns!(), local_name!("class")),
+                        Attribute {
+                            prefix: None,
+                            value: "anchor".to_string(),
+                        },
+                    ),
+                    (
+                        ExpandedName::new(ns!(), local_name!("href")),
+                        Attribute {
+                            prefix: None,
+                            value: format!("#{}", id),
+                        },
+                    ),
+                ],
+            ));
+        });
+
+    let mut bytes = vec![];
+    serialize(&mut bytes, &frag, SerializeOpts::default())?;
+    Ok(String::from_utf8(bytes)?)
+}
+
 fn normalize_name(filename: &str) -> String {
     let decoded = percent_decode_str(filename).decode_utf8_lossy();
     let result = decoded.strip_suffix(".md");
-    String::from(result.unwrap_or(decoded.as_ref()))
+    String::from(result.unwrap_or_else(|| decoded.as_ref()))
 }

templates/graph.html

@@ -0,0 +1,21 @@
+<div id="graph">
+
+</div>
+
+<script src="https://d3js.org/d3.v7.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/d3-dispatch@3"></script>
+<script src="https://cdn.jsdelivr.net/npm/d3-quadtree@3"></script>
+<script src="https://cdn.jsdelivr.net/npm/d3-timer@3"></script>
+<script src="https://cdn.jsdelivr.net/npm/d3-force@3"></script>
+<script>
+    const nodes = {{nodes | json_encode() | safe}};
+    const links = {{links  | json_encode() | safe}};
+</script>
+<script src="/static/graph.js"></script>
+
+<style>
+    #graph {
+        width: 100%;
+        height: 100vh;
+    }
+</style>

templates/graph.js

@@ -0,0 +1,87 @@
+// const nodes = [
+//     {"id": "Alice"},
+//     {"id": "Bob"},
+//     {"id": "Carol"}
+// ];
+//
+// const links = [
+//     {"source": 0, "target": 1}, // Alice → Bob
+//     {"source": 1, "target": 0} // Bob → Carol
+// ];
+
+const graphEl = document.getElementById("graph");
+
+const drag = (simulation) => {
+    function dragstarted(event, d) {
+        if (!event.active) simulation.alphaTarget(0.3).restart();
+        d.fx = d.x;
+        d.fy = d.y;
+    }
+
+    function dragged(event, d) {
+        d.fx = event.x;
+        d.fy = event.y;
+    }
+
+    function dragended(event, d) {
+        if (!event.active) simulation.alphaTarget(0);
+        d.fx = null;
+        d.fy = null;
+    }
+
+    return d3.drag()
+        .on("start", dragstarted)
+        .on("drag", dragged)
+        .on("end", dragended);
+}
+
+
+const simulation = d3.forceSimulation(nodes)
+    .force("link", d3.forceLink(links).id(d => d.id))
+    .force("charge", d3.forceManyBody())
+    .force("x", d3.forceX())
+    .force("y", d3.forceY());
+
+setTimeout(() => {
+    const width = graphEl.clientWidth;
+    const height = graphEl.clientHeight;
+
+    const svg = d3.create("svg")
+        .attr("viewBox", [-width / 2, -height / 2, width, height]);
+
+    const link = svg.append("g")
+        .attr("stroke", "#999")
+        .attr("stroke-opacity", 0.6)
+        .selectAll("line")
+        .data(links)
+        .join("line")
+        .attr("stroke-width", d => Math.sqrt(d.value));
+
+    const node = svg.append("g")
+        .attr("stroke", "#fff")
+        .attr("stroke-width", 1.5)
+        .selectAll("circle")
+        .data(nodes)
+        .join("circle")
+        .attr("r", 5)
+        .attr("fill", "red")
+        .call(drag(simulation));
+
+    node.append("title")
+        .text(d => d.id);
+
+    simulation.on("tick", () => {
+        link
+            .attr("x1", d => d.source.x)
+            .attr("y1", d => d.source.y)
+            .attr("x2", d => d.target.x)
+            .attr("y2", d => d.target.y);
+
+        node
+            .attr("cx", d => d.x)
+            .attr("cy", d => d.y);
+    });
+
+
+    graphEl.appendChild(svg.node());
+}, 0);

templates/main.css

@@ -1,5 +1,15 @@
-body {
+@import url('https://rsms.me/inter/inter.css');
+
+@supports (font-variation-settings: normal) {
+  html { 
+      font-family: 'Inter var', sans-serif;
+      font-feature-settings: "frac", "cpsp", "ss02", "ss03";
+    }
+}
+
+html {
     font-family: 'Inter', sans-serif;
+    line-height: 1.2;
 }
 
 h1, h2, h3, h4, h5 {
@@ -49,12 +59,48 @@ li p {
     margin: 0;
 }
 
+.anchor {
+    text-decoration: none;
+    opacity: .5;
+    color: lightgray;
+}
+
+.anchor:after {
+    content: "#";
+    margin-right: .15em;
+    font-size: 80%;
+}
+
+h2 .anchor:after {
+    content: "##";
+}
+
+h3 .anchor:after {
+    content: "###";
+}
+
+h4 .anchor:after {
+    content: "####";
+}
+
+h5 .anchor:after {
+    content: "#####";
+}
+
+.anchor:visited {
+    color: lightgray;
+}
+
 aside h1 {
     font-size: 16pt;
     text-decoration: underline;
     font-variant: small-caps;
 }
 
+aside h2 {
+    font-size: 12pt;
+}
+
 @media screen and (max-width: 800px) {
     body {
         flex-direction: column-reverse;
@@ -114,6 +160,14 @@ nav li {
     margin: .25em 0;
 }
 
+nav ul a {
+    text-decoration: none;
+}
+
+nav .filepath {
+    text-decoration: underline;
+}
+
 nav .file {
     font-style: italic;
     opacity: .8;
@@ -123,6 +177,26 @@ nav .not-last {
     opacity: .5;
 }
 
+nav .timestamp {
+    opacity: .8;
+}
+
+nav .timestamp:before {
+    content: "[";
+}
+
+nav .timestamp:after {
+    content: "]";
+}
+
+nav .graph-view {
+    text-align: center;
+}
+
+.tag .count {
+    opacity: .5;
+}
+
 footer {
     padding: 1em 0;
     color: gray;
@@ -150,19 +224,15 @@ main footer {
         background: #141414;
         color: #e3e3e3;
     }
-
     a {
         color: #90d7e5;
     }
-
     a:visited {
         color: #3d9bb3;
     }
-
     nav h1 a {
         color: white;
     }
-
     code {
         background: black;
         color: #F8F8FC;

templates/main.html

@@ -19,31 +19,79 @@
     <meta property="og:image" content="/static/favicon.png" />
     <link rel="shortcut icon" href="/static/favicon.png" type="image/x-icon">
     <link href="/static/main.css" rel="stylesheet">
+    <link rel="stylesheet" href="main.css">
 </head>
 
 <body>
     <aside>
         <nav>
             <h1><a href="/">{{garden_title}}</a></h1>
-            <ul>
-                {% for file in files %}
-                    {% if file is containing(".md") %}
-                    <li class="page">
-                        <a href="/{{file}}">
-                            {% for component in file | trim_end_matches(pat=".md") | split(pat=".") %}
-                                {%- if not loop.last -%}
-                                    <span class="not-last">{{component}}.</span>
-                                {%- else -%}
-                                    <span>{{component}}</span>
-                                {%- endif -%}
-                            {% endfor %}
-                        </a>
-                    </li>
-                    {% else %}
-                        <li class="file"><a href="/{{file}}">{{file}}</a></li>
-                    {% endif %}
-                {% endfor %}
-            </ul>
+            {% if recently_changed %}
+                <section>
+                    <h2>Recently changed</h2>
+                    <ul>
+                        {% for file_mtime in recently_changed | slice(end=5) %}
+                            <li class="page">
+                                <a href="/{{file_mtime.0}}">
+                                    <span class="timestamp">{{ file_mtime.1 }}</span>
+                                    <span class="filepath">
+                                        {% for component in file_mtime.0 | trim_end_matches(pat=".md") | split(pat=".") %}
+                                            {%- if not loop.last -%}
+                                                <span class="not-last">{{component}}.</span>
+                                            {%- else -%}
+                                                <span>{{component}}</span>
+                                            {%- endif -%}
+                                        {% endfor %}
+                                    </span>
+                                </a>
+                            </li>
+                        {% endfor %}
+                    </ul>
+                </section>
+            {% endif %}
+            <section>
+                <h2>All entries</h2>
+                <ul>
+                    {% for file in files %}
+                        {% if file is containing(".md") %}
+                            <li class="page">
+                                <a href="/{{file}}">
+                                    <span class="filepath">
+                                        {% for component in file | trim_end_matches(pat=".md") | split(pat=".") %}
+                                            {%- if not loop.last -%}
+                                                <span class="not-last">{{component}}.</span>
+                                            {%- else -%}
+                                                <span>{{component}}</span>
+                                            {%- endif -%}
+                                        {% endfor %}
+                                    </span>
+                                </a>
+                            </li>
+                        {% else %}
+                            <li class="file"><a href="/{{file}}">{{file}}</a></li>
+                        {% endif %}
+                    {% endfor %}
+                </ul>
+            </section>
+            {% if tags %}
+            <section>
+                <h2>Tags</h2>
+                <ul>
+                    {% for tag_cnt in tags %}
+                        {% if tag_cnt.1 > 1 %}
+                            <li class="tag">
+                                <a href="/{{tag_cnt.0}}">
+                                    <span class="label">#{{tag_cnt.0}}</span> <span class="count">({{tag_cnt.1}})</span>
+                                </a>
+                            </li>
+                        {% endif %}
+                    {% endfor %}
+                </ul>
+            </section>
+            {% endif %}
+            <section class="graph-view">
+                <a href="/!graph">Graph view (beta)</a>
+            </section>
         </nav>
         <footer><a href="https://gitlab.com/tmladek/gardenserver">gardenserver {{version}}</a></footer>
     </aside>