build: further refactor Earthfile & build process

separate strict/release & nightly builds, avoid LOCAL by default
2024-02-03 16:35:51 +01:00 · 2024-02-03 16:35:51 +01:00 · e6862351f9
parent 2da5a28a42
commit e6862351f9
3 changed files with 197 additions and 153 deletions
--- a/.woodpecker.yml
+++ b/.woodpecker.yml
@ -38,21 +38,6 @@ pipeline:
  #     - earthly bootstrap
  #     - earthly +audit

-  gitea:prerelease:
-    image: earthly/earthly:v0.8.1
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-    environment:
-      - FORCE_COLOR=1
-      - EARTHLY_EXEC_CMD="/bin/sh"
-    secrets: [EARTHLY_CONFIGURATION, DOCKER_USER, DOCKER_PASSWORD]
-    commands:
-      - mkdir ~/.earthly && echo "$EARTHLY_CONFIGURATION" > ~/.earthly/config.yaml
-      - earthly bootstrap
-      - earthly +current-changelog
-    when:
-      event: [tag]
-
  appimage:
    image: earthly/earthly:v0.8.1
    volumes:
@ -64,7 +49,8 @@ pipeline:
    commands:
      - mkdir ~/.earthly && echo "$EARTHLY_CONFIGURATION" > ~/.earthly/config.yaml
      - earthly bootstrap
-      - earthly --strict +appimage
+      - mkdir -p dist/
+      - earthly -a '+appimage/*.AppImage' dist/

  appimage:save:
    group: push
@ -83,7 +69,7 @@ pipeline:
      target: /
    secrets: [S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY]

-  appimage:publish:nightly:
+  appimage:nightly:publish:
    group: push
    image: earthly/earthly:v0.8.1
    volumes:
@ -143,24 +129,7 @@ pipeline:
      - earthly --strict --push +docker --tag=$CI_COMMIT_TAG
    when:
      event: [tag]
-
-  gitea:release:
-    group: push
-    image: woodpeckerci/plugin-gitea-release
-    settings:
-      base_url: https://git.thm.place
-      files:
-        - "dist/*.AppImage"
-        - "dist/*.asc"
-        - webext/web-ext-artifacts/*.zip
-      checksum: sha512
-      api_key:
-        from_secret: woodpecker_api_key
-      target: main
-      note: CHANGELOG_CURRENT.md
-    when:
-      event: [tag]
-
+  
  jslib:publish:
    group: push
    image: earthly/earthly:v0.8.1
@ -176,3 +145,52 @@ pipeline:
      - earthly --strict --push --secret NPM_TOKEN +publish-js-all
    when:
      branch: [main]
+  
+  gitea:prerelease:
+    image: earthly/earthly:v0.8.1
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      - FORCE_COLOR=1
+      - EARTHLY_EXEC_CMD="/bin/sh"
+    secrets: [ EARTHLY_CONFIGURATION, DOCKER_USER, DOCKER_PASSWORD ]
+    commands:
+      - mkdir ~/.earthly && echo "$EARTHLY_CONFIGURATION" > ~/.earthly/config.yaml
+      - earthly bootstrap
+      - earthly -a +current-changelog/CHANGELOG_CURRENT.md CHANGELOG_CURRENT.md
+      - rm -rf dist
+    when:
+      event: [ tag ]
+      
+  appimage:release:
+    image: earthly/earthly:v0.8.1
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      - FORCE_COLOR=1
+      - EARTHLY_EXEC_CMD="/bin/sh"
+    secrets: [ EARTHLY_CONFIGURATION, REGISTRY, REGISTRY_USER, REGISTRY_PASSWORD ]
+    commands:
+      - mkdir ~/.earthly && echo "$EARTHLY_CONFIGURATION" > ~/.earthly/config.yaml
+      - earthly bootstrap
+      - mkdir -p dist/
+      - earthly --strict -a '+appimage-signed/*' dist/
+    when:
+      event: [tag]
+      
+  # todo: webext
+
+  gitea:release:
+    group: push
+    image: woodpeckerci/plugin-gitea-release
+    settings:
+      base_url: https://git.thm.place
+      files:
+        - "dist/*"
+      checksum: sha512
+      api_key:
+        from_secret: woodpecker_api_key
+      target: main
+      note: CHANGELOG_CURRENT.md
+    when:
+      event: [tag]
--- a/258
+++ b/258
@ -1,4 +1,88 @@
-VERSION 0.7
+VERSION 0.8
+
+# Base targets
+
+base-rust:
+  FROM rust:bookworm
+  RUN rustup component add clippy
+  RUN curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C /usr/local/cargo/bin
+  RUN cargo install wasm-pack wasm-bindgen-cli && rustup target add wasm32-unknown-unknown
+  RUN cargo install cargo-audit
+  WORKDIR /upend
+  CACHE $HOME/.cargo
+  COPY Cargo.toml Cargo.lock .
+  COPY base/Cargo.toml base/Cargo.toml
+  COPY cli/Cargo.toml cli/Cargo.toml
+  COPY db/Cargo.toml db/Cargo.toml
+  COPY tools/upend_wasm/Cargo.toml tools/upend_wasm/Cargo.toml
+  RUN cargo fetch --locked
+
+base-backend:
+  FROM +base-rust
+  COPY --dir base cli db .
+  COPY --dir tools/upend_wasm tools/
+
+base-node:
+  FROM node:lts-iron
+  RUN npm install -g pnpm
+  WORKDIR /upend
+  CACHE $HOME/.local/share/pnpm
+  COPY +wasmlib/pkg-web tools/upend_wasm/pkg-web
+  COPY +wasmlib/pkg-node tools/upend_wasm/pkg-node
+  COPY tools/upend_js/package.json tools/upend_js/pnpm-lock.yaml tools/upend_js/
+  RUN cd tools/upend_js && rm -rf node_modules && pnpm install --frozen-lockfile
+  COPY webui/package.json webui/pnpm-lock.yaml webui/
+  RUN cd webui && rm -rf node_modules && pnpm install --frozen-lockfile
+  COPY --dir webui webext .
+  COPY --dir tools/upend_js tools/
+
+base-frontend:
+  FROM +base-node
+  COPY +jslib/jslib tools/upend_js
+  WORKDIR webui
+  RUN rm -rf node_modules && pnpm install --frozen-lockfile
+
+# Intermediate targets
+
+upend-bin:
+  FROM +base-backend
+  CACHE target
+  RUN cargo build --release
+  COPY +git-version/version.txt .
+  RUN UPEND_VERSION=$(cat version.txt) cargo build --release
+  SAVE ARTIFACT target/release/upend upend
+
+webui:
+  FROM +base-frontend
+  RUN pnpm build
+  SAVE ARTIFACT dist
+
+wasmlib:
+  FROM --platform=linux/amd64 +base-rust
+  COPY --dir base .
+  COPY --dir tools/upend_wasm tools/
+  WORKDIR tools/upend_wasm
+  CACHE target
+  RUN wasm-pack build --target web --out-dir pkg-web && \
+      wasm-pack build --target nodejs --out-dir pkg-node
+  RUN sed -e 's%"name": "upend_wasm"%"name": "@upnd/wasm-web"%' -i pkg-web/package.json && \
+      sed -e 's%"name": "upend_wasm"%"name": "@upnd/wasm-node"%' -i pkg-node/package.json
+  SAVE ARTIFACT pkg-web AS LOCAL tools/upend_wasm/pkg-web
+  SAVE ARTIFACT pkg-node AS LOCAL tools/upend_wasm/pkg-node
+
+jslib:
+  FROM +base-node
+  WORKDIR tools/upend_js
+  RUN pnpm build
+  SAVE ARTIFACT . jslib 
+
+webext:
+  FROM +base-node
+  WORKDIR webext
+  RUN pnpm build
+  SAVE ARTIFACT web-ext-artifacts/*.zip
+  
+# Final targets

 appimage:
  FROM debian:bookworm
@ -12,7 +96,16 @@ appimage:
  COPY build/AppImageBuilder.yml .
  RUN sed -e "s/latest/$(./AppDir/usr/bin/upend --version | cut -d ' ' -f 2)/" -i AppImageBuilder.yml
  RUN pipx run appimage-builder
-  SAVE ARTIFACT UpEnd* AS LOCAL dist/
+  SAVE ARTIFACT UpEnd*
+
+appimage-signed:
+  FROM alpine
+  RUN apk add gpg gpg-agent
+  RUN --secret GPG_SIGN_KEY echo "$GPG_SIGN_KEY" | gpg --import
+  COPY +appimage/*.AppImage .
+  RUN gpg --detach-sign --sign --armor *.AppImage
+  SAVE ARTIFACT *.AppImage
+  SAVE ARTIFACT *.asc

 docker-minimal:
  FROM debian:bookworm
@ -39,7 +132,7 @@ docker:
  SAVE IMAGE --push upend/upend:$tag

 DOCKER_COMMON:
-  COMMAND
+  FUNCTION
  COPY +upend-bin/upend /usr/bin/upend
  COPY --dir +webui/dist /usr/share/upend/webui
  ENTRYPOINT ["/usr/bin/upend"]
@ -48,40 +141,7 @@ DOCKER_COMMON:
  ENV UPEND_NO_DESKTOP=true
  ENV UPEND_ALLOW_HOST='*'

-upend-bin:
-  FROM +base-rust-backend
-  RUN cargo build --release
-  COPY +git-version/version.txt .
-  RUN UPEND_VERSION=$(cat version.txt) cargo build --release
-  SAVE ARTIFACT target/release/upend upend
-
-webui:
-  FROM +base-frontend
-  RUN pnpm build
-  SAVE ARTIFACT dist
-
-wasmlib:
-  FROM --platform=linux/amd64 +base-rust
-  COPY --dir base .
-  COPY --dir tools/upend_wasm tools/
-  WORKDIR tools/upend_wasm
-  RUN wasm-pack build --target web --out-dir pkg-web && \
-      wasm-pack build --target nodejs --out-dir pkg-node
-  RUN sed -e 's%"name": "upend_wasm"%"name": "@upnd/wasm-web"%' -i pkg-web/package.json && \
-      sed -e 's%"name": "upend_wasm"%"name": "@upnd/wasm-node"%' -i pkg-node/package.json
-  SAVE ARTIFACT pkg-web AS LOCAL tools/upend_wasm/pkg-web
-  SAVE ARTIFACT pkg-node AS LOCAL tools/upend_wasm/pkg-node
-
-jslib:
-  FROM +base-node
-  WORKDIR tools/upend_js
-  RUN pnpm build
-  SAVE ARTIFACT . jslib 
-
-webext:
-  FROM +base-node
-  WORKDIR webext
-  RUN pnpm build
+# CI targets

 lint:
  WAIT
@ -91,7 +151,7 @@ lint:
  END

 lint-backend:
-  FROM +base-rust-backend
+  FROM +base-backend
  RUN cargo clippy --workspace

 lint-frontend:
@ -110,7 +170,7 @@ audit:
  END

 audit-backend:
-  FROM +base-rust-backend
+  FROM +base-backend
  RUN cargo audit --workspace

 audit-frontend:
@ -119,27 +179,20 @@ audit-frontend:

 test:
  WAIT
-    BUILD +test-jslib
    BUILD +test-backend
+    BUILD +test-jslib
  END

+test-backend:
+  FROM +base-backend
+  RUN cargo nextest run --workspace
+
 test-jslib:
  FROM +base-node
  WORKDIR tools/upend_js
  RUN pnpm build && pnpm test

-test-backend:
-  FROM +base-rust-backend
-  RUN cargo nextest run --workspace
-
-appimage-signed:
-  FROM alpine
-  RUN apk add gpg gpg-agent
-  RUN --secret GPG_SIGN_KEY echo "$GPG_SIGN_KEY" | gpg --import
-  COPY +appimage/*.AppImage .
-  RUN gpg --detach-sign --sign --armor *.AppImage
-  SAVE ARTIFACT *.AppImage
-  SAVE ARTIFACT *.asc
+# Deployment targets

 deploy-appimage-nightly:
  FROM alpine
@ -152,22 +205,13 @@ deploy-appimage-nightly:
    chmod 600 $HOME/.ssh/*
  COPY +appimage-signed/* .
  RUN --push scp -v *.AppImage *.asc mainsite:releases/nightly
-
-NPM_PUBLISH:
-  COMMAND
-  ARG pkg_name
-  IF [ "`npm view $pkg_name version`" != "`node -p \"require('./package.json').version\"`" ]
-    RUN echo "Publishing $pkg_name to npm..."
-    RUN --push npm publish --access public
-  ELSE
-    RUN echo "Nothing to do for $pkg_name."
+  
+publish-js-all:
+  WAIT
+    BUILD +publish-js-wasm
+    BUILD +publish-js-lib
  END

-base-npm-publish:
-  FROM +base-node
-  RUN --secret NPM_TOKEN echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > $HOME/.npmrc
-  COPY +jslib/jslib tools/upend_js
-
 publish-js-lib:
  FROM +base-npm-publish
  WORKDIR /upend/tools/upend_js
@ -180,70 +224,48 @@ publish-js-wasm:
  WORKDIR /upend/tools/upend_wasm/pkg-node
  DO +NPM_PUBLISH --pkg_name=@upnd/wasm-node

-publish-js-all:
-  BUILD +publish-js-wasm
-  BUILD +publish-js-lib
+base-npm-publish:
+  FROM +base-node
+  RUN --secret NPM_TOKEN echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > $HOME/.npmrc
+  COPY +jslib/jslib tools/upend_js
+
+NPM_PUBLISH:
+  FUNCTION
+  ARG pkg_name
+  IF --no-cache [ "`npm view $pkg_name version`" != "`node -p \"require('./package.json').version\"`" ]
+    RUN echo "Publishing $pkg_name to npm..."
+    RUN --push npm publish --access public
+  ELSE
+    RUN echo "Nothing to do for $pkg_name."
+  END
+
+# Utility targets

 git-version:
-  LOCALLY
-  RUN ./build/get_version.sh | tee /tmp/upend_version.txt
+  FROM debian:bookworm
+  RUN apt-get update && \
+    apt-get -y install git && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+  COPY build/get_version.sh build/get_version.sh
+  COPY .git .git
+  RUN ./build/get_version.sh > /tmp/upend_version.txt && cat /tmp/upend_version.txt
  SAVE ARTIFACT /tmp/upend_version.txt version.txt
-  RUN rm /tmp/upend_version.txt

 changelog:
  FROM orhunp/git-cliff
  COPY .git .git
-  RUN git-cliff -o CHANGELOG.md
+  RUN git-cliff --bump -o CHANGELOG.md
  SAVE ARTIFACT CHANGELOG.md

 current-changelog:
-  FROM +changelog
+  FROM orhunp/git-cliff
+  COPY .git .git
  RUN git-cliff --current -o CHANGELOG_CURRENT.md
-  SAVE ARTIFACT CHANGELOG_CURRENT.md AS LOCAL CHANGELOG_CURRENT.md
+  SAVE ARTIFACT CHANGELOG_CURRENT.md

 update-changelog:
  LOCALLY
  COPY +changelog/CHANGELOG.md .
-  RUN git add CHANGELOG.md && git commit -m "Update CHANGELOG"
-  RUN --push git push
-
-base-rust:
-  FROM rust:bookworm
-  RUN rustup component add clippy
-  RUN curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C /usr/local/cargo/bin
-  RUN cargo install wasm-pack wasm-bindgen-cli && rustup target add wasm32-unknown-unknown
-  RUN cargo install cargo-audit
-  WORKDIR /upend
-  CACHE $HOME/.cargo
-  COPY Cargo.toml Cargo.lock .
-  COPY base/Cargo.toml base/Cargo.toml
-  COPY cli/Cargo.toml cli/Cargo.toml
-  COPY db/Cargo.toml db/Cargo.toml
-  COPY tools/upend_wasm/Cargo.toml tools/upend_wasm/Cargo.toml
-  RUN cargo fetch --locked
-
-base-rust-backend:
-  FROM +base-rust
-  COPY --dir base cli db .
-  COPY --dir tools/upend_wasm tools/
-  CACHE target
-
-base-node:
-  FROM node:lts-iron
-  RUN npm install -g pnpm
-  CACHE $HOME/.local/share/pnpm
-  WORKDIR /upend
-  COPY +wasmlib/pkg-web tools/upend_wasm/pkg-web
-  COPY +wasmlib/pkg-node tools/upend_wasm/pkg-node
-  COPY tools/upend_js/package.json tools/upend_js/pnpm-lock.yaml tools/upend_js/
-  RUN cd tools/upend_js && pnpm install --frozen-lockfile --force
-  COPY webui/package.json webui/pnpm-lock.yaml webui/
-  RUN cd webui && pnpm install --frozen-lockfile
-  COPY --dir webui webext .
-  COPY --dir tools/upend_js tools/
-
-base-frontend:
-  FROM +base-node
-  COPY +jslib/jslib tools/upend_js
-  WORKDIR webui
-  RUN pnpm install --frozen-lockfile --force
+  RUN git add CHANGELOG.md && git commit -m "release: Update CHANGELOG"
+  RUN --push git push
--- a/build/get_version.sh
+++ b/build/get_version.sh
@ -1,5 +1,9 @@
 #!/bin/sh

+which git > /dev/null || {
+  echo "git not found"
+  exit 1
+}
 git_tag=$(git describe --tags --exact-match HEAD 2>/dev/null)

 if [ -z "$git_tag" ]; then