upend/webui/src/components/display/blobs/AudioViewer.svelte

<script lang="ts">
  import { debounce, throttle } from "lodash";
  import { onMount } from "svelte";
  import type { IValue } from "@upnd/upend/types";
  import type WaveSurfer from "wavesurfer.js";
  import type { Region, RegionParams } from "wavesurfer.js/src/plugin/regions";
  import api from "../../../lib/api";
  import { TimeFragment } from "../../../util/fragments/time";
  import Icon from "../../utils/Icon.svelte";
  import Selector from "../../utils/Selector.svelte";
  import UpObject from "../../display/UpObject.svelte";
  import Spinner from "../../utils/Spinner.svelte";
  import IconButton from "../../../components/utils/IconButton.svelte";
  import LabelBorder from "../../../components/utils/LabelBorder.svelte";
  import { i18n } from "../../../i18n";
  import { ATTR_LABEL } from "@upnd/upend/constants";
  import debug from "debug";
  const dbg = debug("kestrel:AudioViewer");

  export let address: string;
  export let detail: boolean;

  let editable = false;

  let containerEl: HTMLDivElement;
  let timelineEl: HTMLDivElement;
  let loaded = false;

  let wavesurfer: WaveSurfer;

  // Zoom handling
  let zoom = 1;
  const setZoom = throttle((level: number) => {
    wavesurfer.zoom(level);
  }, 250);
  $: if (zoom && wavesurfer) setZoom(zoom);

  // Annotations
  const DEFAULT_ANNOTATION_COLOR = "#cb4b16";

  type UpRegion = Region & { data: IValue };
  let currentAnnotation: UpRegion | undefined;

  async function loadAnnotations() {
    const entity = await api.fetchEntity(address);
    entity.backlinks
      .filter((e) => e.attribute == "ANNOTATES")
      .forEach(async (e) => {
        const annotation = await api.fetchEntity(e.entity);
        if (annotation.get("W3C_FRAGMENT_SELECTOR")) {
          const fragment = TimeFragment.parse(
            String(annotation.get("W3C_FRAGMENT_SELECTOR")),
          );
          if (fragment) {
            wavesurfer.addRegion({
              id: `ws-region-${e.entity}`,
              color: annotation.get("COLOR") || DEFAULT_ANNOTATION_COLOR,
              attributes: {
                "upend-address": annotation.address,
                label: annotation.get(ATTR_LABEL),
              },
              data: (annotation.attr["NOTE"] || [])[0]?.value,
              ...fragment,
            } as RegionParams);
          }
        }
      });
  }

  $: if (wavesurfer) {
    if (editable) {
      wavesurfer.enableDragSelection({ color: DEFAULT_ANNOTATION_COLOR });
    } else {
      wavesurfer.disableDragSelection();
    }
    Object.values(wavesurfer.regions.list).forEach((region) => {
      region.update({ drag: editable, resize: editable });
    });
  }

  async function updateAnnotation(region: Region) {
    dbg("Updating annotation %o", region);
    let entity = region.attributes["upend-address"];

    // Newly created
    if (!entity) {
      let [_, newEntity] = await api.putEntry({
        entity: {
          t: "Uuid",
        },
      });
      entity = newEntity;

      const nextAnnotationIndex = Object.values(wavesurfer.regions.list).length;
      const label = `Annotation #${nextAnnotationIndex}`;

      region.update({
        attributes: { label },
        // incorrect types, `update()` does take `attributes`
        // eslint-disable-next-line @typescript-eslint/no-explicit-any
      } as any);
    }

    if (region.attributes["label"]) {
      await api.putEntityAttribute(entity, ATTR_LABEL, {
        t: "String",
        c: region.attributes["label"],
      });
    }

    await api.putEntityAttribute(entity, "ANNOTATES", {
      t: "Address",
      c: address,
    });

    await api.putEntityAttribute(entity, "W3C_FRAGMENT_SELECTOR", {
      t: "String",
      c: new TimeFragment(region.start, region.end).toString(),
    });

    if (region.color !== DEFAULT_ANNOTATION_COLOR) {
      await api.putEntityAttribute(entity, "COLOR", {
        t: "String",
        c: region.color,
      });
    }

    if (Object.values(region.data).length) {
      await api.putEntityAttribute(entity, "NOTE", region.data as IValue);
    }

    region.update({
      attributes: {
        "upend-address": entity,
      },
      // eslint-disable-next-line @typescript-eslint/no-explicit-any
    } as any);
  }
  const updateAnnotationDebounced = debounce(updateAnnotation, 250);

  async function deleteAnnotation(region: Region) {
    if (region.attributes["upend-address"]) {
      await api.deleteEntry(region.attributes["upend-address"]);
    }
  }

  let rootEl: HTMLElement;
  onMount(async () => {
    const WaveSurfer = await import("wavesurfer.js");
    const TimelinePlugin = await import("wavesurfer.js/src/plugin/timeline");
    const RegionsPlugin = await import("wavesurfer.js/src/plugin/regions");
    const timelineColor = getComputedStyle(
      document.documentElement,
    ).getPropertyValue("--foreground");

    wavesurfer = WaveSurfer.default.create({
      container: containerEl,
      waveColor: "#dc322f",
      progressColor: "#991c1a",
      responsive: true,
      backend: "MediaElement",
      mediaControls: true,
      normalize: true,
      xhr: { cache: "force-cache" },
      plugins: [
        TimelinePlugin.default.create({
          container: timelineEl,
          primaryColor: timelineColor,
          primaryFontColor: timelineColor,
          secondaryColor: timelineColor,
          secondaryFontColor: timelineColor,
        }),
        RegionsPlugin.default.create({}),
      ],
    });

    wavesurfer.on("ready", () => {
      dbg("wavesurfer ready");

      loaded = true;
      loadAnnotations();
    });

    wavesurfer.on("region-created", async (region: UpRegion) => {
      dbg("wavesurfer region-created", region);

      // Updating here, because if `drag` and `resize` are passed during adding,
      // updating no longer works.
      region.update({ drag: editable, resize: editable });

      // If the region was created from the UI
      if (!region.attributes["upend-address"]) {
        await updateAnnotation(region);
        // currentAnnotation = region;
      }
    });

    wavesurfer.on("region-updated", (region: UpRegion) => {
      // dbg("wavesurfer region-updated", region);

      currentAnnotation = region;
    });

    wavesurfer.on("region-update-end", (region: UpRegion) => {
      dbg("wavesurfer region-update-end", region);

      updateAnnotation(region);
      currentAnnotation = region;
    });

    wavesurfer.on("region-removed", (region: UpRegion) => {
      dbg("wavesurfer region-removed", region);

      currentAnnotation = null;
      deleteAnnotation(region);
    });

    // wavesurfer.on("region-in", (region: UpRegion) => {
    //   dbg("wavesurfer region-in", region);

    //   currentAnnotation = region;
    // });

    // wavesurfer.on("region-out", (region: UpRegion) => {
    //   dbg("wavesurfer region-out", region);

    //   if (currentAnnotation.id === region.id) {
    //     currentAnnotation = undefined;
    //   }
    // });

    wavesurfer.on("region-click", (region: UpRegion, _ev: MouseEvent) => {
      dbg("wavesurfer region-click", region);

      currentAnnotation = region;
    });

    wavesurfer.on("region-dblclick", (region: UpRegion, _ev: MouseEvent) => {
      dbg("wavesurfer region-dblclick", region);

      currentAnnotation = region;
      setTimeout(() => wavesurfer.setCurrentTime(region.start));
    });

    try {
      const peaksReq = await fetch(
        `${api.apiUrl}/thumb/${address}?mime=audio&type=json`,
      );
      const peaks = await peaksReq.json();
      wavesurfer.load(`${api.apiUrl}/raw/${address}`, peaks.data);
    } catch (e) {
      console.warn(`Failed to load peaks: ${e}`);
      const entity = await api.fetchEntity(address);
      if (
        (parseInt(String(entity.get("FILE_SIZE"))) || 0) < 20_000_000 ||
        confirm(
          $i18n.t(
            "File is large (>20 MiB) and UpEnd failed to load waveform from server. Generating the waveform locally may slow down your browser. Do you wish to proceed anyway?",
          ),
        )
      ) {
        console.warn(
          `Failed to load peaks, falling back to client-side render...`,
        );

        wavesurfer.load(`${api.apiUrl}/raw/${address}`);
      }
    }

    const drawBufferThrottled = throttle(() => wavesurfer.drawBuffer(), 200);
    const resizeObserver = new ResizeObserver((_entries) => {
      drawBufferThrottled();
    });
    resizeObserver.observe(rootEl);
  });
</script>

<div class="audio" class:editable bind:this={rootEl}>
  {#if !loaded}
    <Spinner centered />
  {/if}
  {#if loaded}
    <header>
      <IconButton
        name="edit"
        title={$i18n.t("Toggle Edit Mode")}
        on:click={() => (editable = !editable)}
        active={editable}
      >
        {$i18n.t("Annotate")}
      </IconButton>
      <div class="zoom">
        <Icon name="zoom-out" />
        <input type="range" min="1" max="50" bind:value={zoom} />
        <Icon name="zoom-in" />
      </div>
    </header>
  {/if}
  <div
    class="wavesurfer-timeline"
    bind:this={timelineEl}
    class:hidden={!detail}
  />
  <div class="wavesurfer" bind:this={containerEl} />
  {#if currentAnnotation}
    <LabelBorder>
      <span slot="header">{$i18n.t("Annotation")}</span>
      {#if currentAnnotation.attributes["upend-address"]}
        <UpObject
          link
          address={currentAnnotation.attributes["upend-address"]}
        />
      {/if}
      <div class="baseControls">
        <div class="regionControls">
          <div class="start">
            Start: <input
              type="number"
              value={Math.round(currentAnnotation.start * 100) / 100}
              disabled={!editable}
              on:input={(ev) => {
                currentAnnotation.update({
                  start: parseInt(ev.currentTarget.value),
                });
                updateAnnotationDebounced(currentAnnotation);
              }}
            />
          </div>
          <div class="end">
            End: <input
              type="number"
              value={Math.round(currentAnnotation.end * 100) / 100}
              disabled={!editable}
              on:input={(ev) => {
                currentAnnotation.update({
                  end: parseInt(ev.currentTarget.value),
                });
                updateAnnotationDebounced(currentAnnotation);
              }}
            />
          </div>
          <div class="color">
            Color: <input
              type="color"
              value={currentAnnotation.color || DEFAULT_ANNOTATION_COLOR}
              disabled={!editable}
              on:input={(ev) => {
                currentAnnotation.update({ color: ev.currentTarget.value });
                updateAnnotation(currentAnnotation);
              }}
            />
          </div>
        </div>
        {#if editable}
          <div class="existControls">
            <IconButton
              outline
              name="trash"
              on:click={() => currentAnnotation.remove()}
            />
            <!-- <div class="button">
            <Icon name="check" />
          </div> -->
          </div>
        {/if}
      </div>
      <div class="content">
        {#key currentAnnotation}
          <Selector
            types={["String", "Address"]}
            initial={currentAnnotation.data}
            disabled={!editable}
            on:input={(ev) => {
              currentAnnotation.update({ data: ev.detail });
              updateAnnotation(currentAnnotation);
            }}
          />
        {/key}
      </div>
    </LabelBorder>
  {/if}
</div>

<style lang="scss">
  @use "../../../styles/colors";

  .audio {
    width: 100%;
  }

  header {
    display: flex;
    justify-content: space-between;
    & > * {
      flex-basis: 50%;
    }
    .zoom {
      display: flex;
      align-items: baseline;
      input {
        flex-grow: 1;
        margin: 0 0.5em 1em 0.5em;
      }
    }
  }

  .baseControls,
  .content {
    margin: 0.5em 0;
  }

  .baseControls,
  .regionControls,
  .existControls {
    display: flex;
    gap: 0.5em;
  }

  .baseControls {
    justify-content: space-between;
  }

  .regionControls div {
    display: flex;
    align-items: center;
    gap: 0.25em;
  }

  input[type="number"] {
    width: 6em;
  }

  .hidden {
    display: none;
  }

  :global(.audio:not(.editable) .wavesurfer-handle) {
    display: none;
  }

  :global(.wavesurfer-handle) {
    background: var(--foreground-lightest) !important;
  }

  :global(.wavesurfer-region) {
    opacity: 0.5;
  }
</style>