[ui] first version of audio annotations

TODOs:
- fix wavesurfer types
- fix selector not updating when value prop changes
- only allow annotation editing in edit mode
- allow addresses to be annotation values as well

webui/src/components/display/blobs/AudioViewer.svelte

@@ -1,8 +1,17 @@
 <script lang="ts">
-  import { throttle } from "lodash";
+  import { debounce, throttle } from "lodash";
   import { onMount } from "svelte";
   import type WaveSurfer from "wavesurfer.js";
+  import type { Region, RegionParams } from "wavesurfer.js/src/plugin/regions";
+  import {
+    deleteEntry,
+    fetchEntity,
+    putEntityAttribute,
+    putEntry,
+  } from "../../../lib/api";
+  import { TimeFragment } from "../../../util/fragments/time";
   import Icon from "../../utils/Icon.svelte";
+  import Selector from "../../utils/Selector.svelte";
   import Spinner from "../../utils/Spinner.svelte";
 
   export let address: string;
@@ -13,17 +22,116 @@
   let loaded = false;
 
   let wavesurfer: WaveSurfer;
+  // Trigger re-render by calling `drawBuffer()` on `detail` state change.
   $: if (wavesurfer) wavesurfer.drawBuffer() && detail;
 
+  // Zoom handling
   $: zoom = detail ? 1 : undefined;
   const setZoom = throttle((level: number) => {
     wavesurfer.zoom(level);
   }, 250);
   $: if (zoom && wavesurfer) setZoom(zoom);
 
+  // Annotations
+  const DEFAULT_ANNOTATION_COLOR = "#cb4b16";
+
+  let currentAnnotation: Region | undefined;
+  $: currentAnnotationIndex =
+    Array.from(regions)
+      .sort((a, b) => a.start - b.start)
+      .findIndex((r) => r.id === currentAnnotation.id) + 1;
+
+  async function loadAnnotations() {
+    const entity = await fetchEntity(address);
+    entity.backlinks
+      .filter((e) => e.attribute == "ANNOTATES")
+      .forEach(async (e) => {
+        const annotation = await fetchEntity(e.entity);
+        if (annotation.get("W3C_FRAGMENT_SELECTOR")) {
+          const fragment = TimeFragment.parse(
+            String(annotation.get("W3C_FRAGMENT_SELECTOR"))
+          );
+          if (fragment) {
+            wavesurfer.addRegion({
+              id: `ws-region-${e.entity}`,
+              color: annotation.get("COLOR") || DEFAULT_ANNOTATION_COLOR,
+              attributes: { "upend-id": annotation.address },
+              data: (annotation.attr["LBL"] || [])[0]?.value,
+              ...fragment,
+            } as RegionParams);
+          }
+        }
+      });
+  }
+
+  async function createAnnotation(region: Region) {
+    const [_, uuid] = await putEntry({
+      entity: {
+        t: "Uuid",
+      },
+    });
+
+    // incorrect types, `update()` does take `attributes`
+    region.update({ attributes: { "upend-id": uuid } } as any);
+
+    await putEntry([
+      {
+        entity: uuid,
+        attribute: "ANNOTATES",
+        value: {
+          t: "Address",
+          c: address,
+        },
+      },
+      {
+        entity: uuid,
+        attribute: "W3C_FRAGMENT_SELECTOR",
+        value: {
+          t: "String",
+          c: new TimeFragment(region.start, region.end).toString(),
+        },
+      },
+    ]);
+  }
+
+  $: if (wavesurfer) {
+    if (detail) {
+      wavesurfer.enableDragSelection({ color: DEFAULT_ANNOTATION_COLOR });
+    } else {
+      wavesurfer.disableDragSelection();
+    }
+    regions.forEach((region) =>
+      region.update({ drag: detail, resize: detail })
+    );
+  }
+
+  async function updateAnnotation(region: Region) {
+    const entity = region.attributes["upend-id"];
+    await putEntityAttribute(entity, "W3C_FRAGMENT_SELECTOR", {
+      t: "String",
+      c: new TimeFragment(region.start, region.end).toString(),
+    });
+
+    await putEntityAttribute(entity, "LBL", region.data);
+
+    await putEntityAttribute(entity, "COLOR", {
+      t: "String",
+      c: region.color,
+    });
+  }
+  const updateAnnotationDebounced = debounce(updateAnnotation, 250);
+
+  async function deleteAnnotation(region: Region) {
+    // TODO - what if there's no id?
+    await deleteEntry(region.attributes["upend-id"]);
+  }
+
+  const regions = new Set<Region>();
+
   onMount(async () => {
     const WaveSurfer = await import("wavesurfer.js");
     const TimelinePlugin = await import("wavesurfer.js/src/plugin/timeline");
+    const RegionsPlugin = await import("wavesurfer.js/src/plugin/regions");
     const timelineColor = getComputedStyle(
       document.documentElement
     ).getPropertyValue("--foreground");
@@ -44,11 +152,52 @@
           secondaryColor: timelineColor,
           secondaryFontColor: timelineColor,
         }),
+        RegionsPlugin.default.create({}),
       ],
     });
+
     wavesurfer.on("ready", () => {
       loaded = true;
+      loadAnnotations();
     });
+
+    wavesurfer.on("region-created", (region: Region) => {
+      regions.add(region);
+      if (!region.attributes["upend-id"]) {
+        createAnnotation(region);
+        currentAnnotation = region;
+      }
+    });
+
+    wavesurfer.on("region-updated", (region: Region) => {
+      currentAnnotation = region;
+    });
+
+    wavesurfer.on("region-update-end", (region: Region) => {
+      updateAnnotation(region);
+      currentAnnotation = region;
+    });
+
+    wavesurfer.on("region-removed", (region: Region) => {
+      deleteAnnotation(region);
+      regions.delete(region);
+    });
+
+    wavesurfer.on("region-in", (region: Region) => {
+      currentAnnotation = region;
+    });
+
+    wavesurfer.on("region-out", (region: Region) => {
+      if (currentAnnotation.id === region.id) {
+        currentAnnotation = undefined;
+      }
+    });
+
+    wavesurfer.on("region-dblclick", (region: Region, ev: MouseEvent) => {
+      currentAnnotation = region;
+      setTimeout(() => wavesurfer.setCurrentTime(region.start));
+    });
+
     wavesurfer.load(`api/raw/${address}`);
   });
 </script>
@@ -74,9 +223,76 @@
     class:hidden={!detail}
   />
   <div class="wavesurfer" bind:this={containerEl} />
+  {#if currentAnnotation}
+    <section class="annotationEditor labelborder">
+      <header><h3>Annotation #{currentAnnotationIndex}</h3></header>
+      <div class="baseControls">
+        <div class="regionControls">
+          <div class="start">
+            Start: <input
+              type="number"
+              value={Math.round(currentAnnotation.start * 100) / 100}
+              on:input={(ev) => {
+                currentAnnotation.update({
+                  start: parseInt(ev.currentTarget.value),
+                });
+                updateAnnotationDebounced(currentAnnotation);
+              }}
+            />
+          </div>
+          <div class="end">
+            End: <input
+              type="number"
+              value={Math.round(currentAnnotation.end * 100) / 100}
+              on:input={(ev) => {
+                currentAnnotation.update({
+                  end: parseInt(ev.currentTarget.value),
+                });
+                updateAnnotationDebounced(currentAnnotation);
+              }}
+            />
+          </div>
+          <div class="color">
+            Color: <input
+              type="color"
+              value={currentAnnotation.color || DEFAULT_ANNOTATION_COLOR}
+              on:input={(ev) => {
+                currentAnnotation.update({ color: ev.currentTarget.value });
+                updateAnnotation(currentAnnotation);
+              }}
+            />
+          </div>
+        </div>
+        <div class="existControls">
+          <div class="button" on:click={() => currentAnnotation.remove()}>
+            <Icon name="trash" />
+          </div>
+          <!-- <div class="button">
+            <Icon name="check" />
+          </div> -->
+        </div>
+      </div>
+      <div class="content">
+        {#key currentAnnotation}
+          <Selector
+            type="value"
+            valueTypes={["String"]}
+            value={currentAnnotation.data}
+            on:input={(ev) => {
+              currentAnnotation.update({ data: ev.detail });
+              updateAnnotation(currentAnnotation);
+            }}
+          />
+        {/key}
+      </div>
+    </section>
+  {/if}
 </div>
 
 <style lang="scss">
+  @use "../../../styles/colors";
+  @use "../../util";
+
   .audio {
     width: 100%;
   }
@@ -89,7 +305,7 @@
     }
     .zoom {
       display: flex;
-      align-items: center;
+      align-items: baseline;
       input {
         flex-grow: 1;
         margin: 0 0.5em 1em 0.5em;
@@ -97,7 +313,42 @@
     }
   }
 
+  .annotationEditor {
+    & > * {
+      margin: 0.5em 0;
+    }
+
+    .baseControls,
+    .regionControls,
+    .existControls {
+      display: flex;
+      gap: 0.5em;
+    }
+
+    .baseControls {
+      justify-content: space-between;
+    }
+
+    .regionControls div {
+      display: flex;
+      align-items: center;
+      gap: 0.25em;
+    }
+
+    input[type="number"] {
+      width: 6em;
+    }
+  }
+
   .hidden {
     display: none;
   }
+
+  :global(.wavesurfer-handle) {
+    background: var(--foreground-lightest) !important;
+  }
+
+  :global(.wavesurfer-region) {
+    opacity: 0.5;
+  }
 </style>

webui/src/components/display/blobs/FragmentViewer.svelte

@@ -3,7 +3,7 @@
   import Spinner from "../../utils/Spinner.svelte";
   export let address: string;
   export let detail: boolean;
-  import { xywh } from "../../../util/xywh";
+  import { xywh } from "../../../util/fragments/xywh";
   import UpLink from "../UpLink.svelte";
 
   const { entity } = useEntity(address);
@@ -43,7 +43,10 @@
     justify-content: center;
   }
 
-  img.imageLoaded {
-    border: 2px dashed colors.$yellow;
+  img {
+    max-width: 100%;
+    &.imageLoaded {
+      border: 2px dashed colors.$yellow;
+    }
   }
 </style>

webui/src/util/fragments/time.ts

@@ -0,0 +1,29 @@
+/**
+ * Both `start` and `end` are in seconds.
+ */
+export class TimeFragment {
+  start: number | null;
+  end: number | null;
+
+  constructor(start: number, end: number) {
+    this.start = start;
+    this.end = end;
+  }
+
+  public static parse(fragment: string): TimeFragment {
+    if (!fragment.startsWith("t=")) {
+      return undefined;
+    }
+    const data = fragment.substring("t=".length);
+    try {
+      const [start, end] = data.split(",").map((str) => parseFloat(str));
+      return new TimeFragment(start || null, end || null);
+    } catch {
+      return undefined;
+    }
+  }
+
+  public toString(): string {
+    return `t=${this.start || ""},${this.end || ""}`;
+  }
+}