[ui] add "Fragment Viewer"

webui/src/components/display/BlobPreview.svelte

@@ -2,6 +2,7 @@
   import { useEntity } from "../../lib/entity";
   import Spinner from "../utils/Spinner.svelte";
   import AudioViewer from "./blobs/AudioViewer.svelte";
+  import FragmentViewer from "./blobs/FragmentViewer.svelte";
   import ImageViewer from "./blobs/ImageViewer.svelte";
   import ModelViewer from "./blobs/ModelViewer.svelte";
   import TextViewer from "./blobs/TextViewer.svelte";
@@ -27,8 +28,10 @@
     mimeType?.startsWith("model") ||
     $entity?.identify().some((l) => l.endsWith(".stl"));
   $: web = $entityInfo?.t == "Url";
+  $: fragment = Boolean($entity?.get("ANNOTATES"));
 
-  $: handled = audio || video || image || text || pdf || model || web;
+  $: handled =
+    audio || video || image || text || pdf || model || web || fragment;
 
   let imageLoaded = null;
 </script>
@@ -86,6 +89,9 @@
         on:error={() => (handled = false)}
       />
     {/if}
+    {#if fragment}
+      <FragmentViewer {address} {detail} />
+    {/if}
   </div>
 {/if}
 

webui/src/components/display/blobs/FragmentViewer.svelte

@@ -0,0 +1,49 @@
+<script lang="ts">
+  import { useEntity } from "../../../lib/entity";
+  import Spinner from "../../utils/Spinner.svelte";
+  export let address: string;
+  export let detail: boolean;
+  import { xywh } from "../../../util/xywh";
+  import UpLink from "../UpLink.svelte";
+
+  const { entity } = useEntity(address);
+
+  $: objectAddress = String($entity?.get("ANNOTATES") || "");
+
+  let imageLoaded = false;
+</script>
+
+<div class="fragment-viewer">
+  {#if !imageLoaded}
+    <Spinner />
+  {/if}
+  {#if $entity}
+    <UpLink to={{ entity: objectAddress }}>
+      <img
+        class="preview-image"
+        class:imageLoaded
+        src="api/{detail ? 'raw' : 'thumb'}/{objectAddress}#{$entity?.get(
+          'W3C_FRAGMENT_SELECTOR'
+        )}"
+        use:xywh
+        alt={address}
+        on:load={() => (imageLoaded = true)}
+        draggable="false"
+      />
+    </UpLink>
+  {/if}
+</div>
+
+<style lang="scss">
+  @use "../../../styles/colors";
+
+  .fragment-viewer {
+    width: 100%;
+    display: flex;
+    justify-content: center;
+  }
+
+  img.imageLoaded {
+    border: 2px dashed colors.$yellow;
+  }
+</style>

webui/src/util/xywh.ts

@@ -0,0 +1,169 @@
+// Code adapted from `xywh.js` by Thomas Steiner
+// https://github.com/tomayac/xywh.js
+
+export type MediaFragment = (
+  | {
+      mediaItem: HTMLImageElement;
+      mediaType: "img";
+    }
+  | {
+      mediaItem: HTMLVideoElement;
+      mediaType: "video";
+    }
+) & {
+  unit: string;
+  x: number;
+  y: number;
+  w: number;
+  h: number;
+};
+
+export function xywh(mediaItem: HTMLImageElement | HTMLVideoElement) {
+  const source = mediaItem.src || mediaItem.currentSrc;
+  // See http://www.w3.org/TR/media-frags/#naming-space
+  const xywhRegEx =
+    /[#&\?]xywh\=(pixel\:|percent\:)?([\d\.]+),([\d\.]+),([\d\.]+),([\d\.]+)/;
+  const match = xywhRegEx.exec(source);
+  if (match) {
+    const mediaFragment = {
+      mediaItem: mediaItem,
+      mediaType: mediaItem.nodeName.toLowerCase(),
+      unit: match[1] ? match[1] : "pixel:",
+      x: parseFloat(match[2]),
+      y: parseFloat(match[3]),
+      w: parseFloat(match[4]),
+      h: parseFloat(match[5]),
+    } as MediaFragment;
+    if (mediaFragment.mediaType === "img") {
+      addImageLoadListener(mediaFragment);
+    } else {
+      addVideoLoadListener(mediaFragment);
+    }
+  }
+}
+
+/**
+ * Applies the media fragment when the image has loaded. We need the image's
+ * original width and height.
+ */
+function addImageLoadListener(mediaFragment: MediaFragment) {
+  const mediaItem = mediaFragment.mediaItem;
+  const onload = function () {
+    applyFragment(mediaFragment);
+    // Removes the load listener from  the image, so that it doesn't fire
+    // again when we set the image's @src to a transparent 1x1 GIF, but only
+    // once when the initial image has fully loaded
+    mediaItem.removeEventListener("load", onload);
+    // Base64-encoded transparent 1x1 pixel GIF
+    mediaItem.src =
+      "data:image/gif;base64,R0lGODlhAQABAPAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==";
+  };
+  mediaItem.addEventListener("load", onload);
+}
+
+/**
+ * Applies the media fragment when all metadata of the video have loaded. We
+ * need the video's original width and height.
+ */
+function addVideoLoadListener(mediaFragment: MediaFragment) {
+  mediaFragment.mediaItem.addEventListener("loadedmetadata", function () {
+    applyFragment(mediaFragment);
+  });
+}
+
+/**
+ * Applies the spatial media fragment:
+ *
+ * For images, we replace the @src attribute with a transparent GIF data URL,
+ * resize the image to match the fragment's dimensions, and set the image's
+ * CSS background-image according to the fragment's x and y values.
+ *
+ * For videos, we wrap the video in a <div> wrapper of the fragment's
+ * dimensions, set its CSS overflow value to "hidden", and apply a CSS3
+ * 2D transformation according to the fragment's x and y values.
+ */
+function applyFragment(fragment: MediaFragment) {
+  let x: string, y: string, w: string, h: string;
+  const originalWidth =
+    fragment.mediaType === "img"
+      ? fragment.mediaItem.width
+      : fragment.mediaItem.videoWidth;
+  const originalHeight =
+    fragment.mediaType === "img"
+      ? fragment.mediaItem.height
+      : fragment.mediaItem.videoHeight;
+  // Unit is pixel:
+  if (fragment.unit === "pixel:") {
+    const scale =
+      fragment.mediaType === "img"
+        ? originalWidth / fragment.mediaItem.naturalWidth
+        : originalWidth / fragment.mediaItem.clientWidth;
+    w = fragment.w * scale + "px";
+    h = fragment.h * scale + "px";
+    x = "-" + fragment.x * scale + "px";
+    y = "-" + fragment.y * scale + "px";
+    // Unit is percent:
+  } else {
+    w = (originalWidth * fragment.w) / 100 + "px";
+    h = (originalHeight * fragment.h) / 100 + "px";
+    x = "-" + (originalWidth * fragment.x) / 100 + "px";
+    y = "-" + (originalHeight * fragment.y) / 100 + "px";
+  }
+  // Media item is a video
+  if (fragment.mediaType === "video") {
+    const wrapper = document.createElement("div");
+    wrapper.style.cssText +=
+      "overflow:hidden;" +
+      "width:" +
+      w +
+      ";" +
+      "height:" +
+      h +
+      ";" +
+      "padding:0;" +
+      "margin:0;" +
+      "border-radius:0;" +
+      "border:none;";
+    fragment.mediaItem.style.cssText +=
+      "transform:translate(" +
+      x +
+      "," +
+      y +
+      ");" +
+      "-webkit-transform:translate(" +
+      x +
+      "," +
+      y +
+      ");";
+    // Evil DOM operations
+    fragment.mediaItem.parentNode.insertBefore(wrapper, fragment.mediaItem);
+    wrapper.appendChild(fragment.mediaItem);
+
+    // We need to manually trigger @autoplay, as DOM access seems to kill it
+    if (fragment.mediaItem.hasAttribute("autoplay")) {
+      fragment.mediaItem.play();
+    }
+    // Media item is an image
+  } else {
+    fragment.mediaItem.style.cssText +=
+      "width:" +
+      w +
+      ";" +
+      "height:" +
+      h +
+      ";" +
+      "background:url(" +
+      fragment.mediaItem.src +
+      ") " + // background-image
+      "no-repeat " + // background-repeat
+      x +
+      " " +
+      y +
+      "; " + // background-position
+      "background-size: " +
+      originalWidth +
+      "px " +
+      originalHeight +
+      "px;";
+  }
+}