From d73ee7ebf5281d174abf61739449f830610010a6 Mon Sep 17 00:00:00 2001 From: Rohan Kumar Date: Wed, 10 Aug 2022 21:30:22 -0700 Subject: [PATCH] Speed up internal jobs - Make xhtml and html alternates the same (we're polygot), cutting static-compression time in half - Make axe-ff run on local files, reducing some overhead. --- Makefile | 9 +++++++++ Makefile.online | 3 ++- scripts/bin/axe-ff | 8 ++++++++ scripts/compress.sh | 2 +- scripts/copy-file-to-xhtml.sh | 12 ++++++++++++ scripts/xhtmlize-single-file.sh | 12 ++---------- 6 files changed, 34 insertions(+), 12 deletions(-) create mode 100755 scripts/bin/axe-ff create mode 100644 scripts/copy-file-to-xhtml.sh diff --git a/Makefile b/Makefile index 6139020..e20bd69 100644 --- a/Makefile +++ b/Makefile @@ -112,6 +112,10 @@ compress: gzip brotli xhtmlize: sh scripts/xhtmlize.sh $(OUTPUT_DIR) +.PHONY: copy-to-xhtml +copy-to-xhtml: + find $(OUTPUT_DIR) -type f -name "*.html" -exec sh scripts/copy-file-to-xhtml.sh {} \; + # save webmentions to a file, don't send yet mentions.json: hugo # gather old version of the site @@ -142,12 +146,14 @@ deploy: deploy-html deploy-gemini .PHONY: deploy-prod deploy-prod: .prepare-deploy @$(MAKE) compress + @$(MAKE) copy-to-xhtml @$(MAKE) deploy .PHONY: deploy-onion deploy-onion: @$(MAKE) WWW_ROOT=/var/www/seirdy.onion HUGO_BASEURL='http://wgq3bd2kqoybhstp77i3wrzbfnsyd27wt34psaja4grqiezqircorkyd.onion/' OUTPUT_DIR=public_onion .prepare-deploy @$(MAKE) WWW_ROOT=/var/www/seirdy.onion HUGO_BASEURL='http://wgq3bd2kqoybhstp77i3wrzbfnsyd27wt34psaja4grqiezqircorkyd.onion/' OUTPUT_DIR=public_onion compress + @$(MAKE) WWW_ROOT=/var/www/seirdy.onion HUGO_BASEURL='http://wgq3bd2kqoybhstp77i3wrzbfnsyd27wt34psaja4grqiezqircorkyd.onion/' OUTPUT_DIR=public_onion copy-to-xhtml @$(MAKE) WWW_ROOT=/var/www/seirdy.onion HUGO_BASEURL='http://wgq3bd2kqoybhstp77i3wrzbfnsyd27wt34psaja4grqiezqircorkyd.onion/' OUTPUT_DIR=public_onion deploy-html # we only deploy html to the staging site @@ -155,16 +161,19 @@ deploy-onion: deploy-staging: @$(MAKE) HUGO_FLAGS=-DF DOMAIN=staging.seirdy.one USER=deploy@seirdy.one OUTPUT_DIR=public_staging .prepare-deploy @$(MAKE) HUGO_FLAGS=-DF DOMAIN=staging.seirdy.one USER=deploy@seirdy.one OUTPUT_DIR=public_staging compress + @$(MAKE) HUGO_FLAGS=-DF DOMAIN=staging.seirdy.one USER=deploy@seirdy.one OUTPUT_DIR=public_staging copy-to-xhtml @$(MAKE) HUGO_FLAGS=-DF DOMAIN=staging.seirdy.one USER=deploy@seirdy.one OUTPUT_DIR=public_staging deploy-html .PHONY: lint-and-deploy-staging lint-and-deploy-staging: @$(MAKE) HUGO_FLAGS='-DF' DOMAIN=staging.seirdy.one USER=deploy@seirdy.one OUTPUT_DIR=public_staging .prepare-deploy @$(MAKE) HUGO_FLAGS='-DF' DOMAIN=staging.seirdy.one USER=deploy@seirdy.one OUTPUT_DIR=public_staging compress lint-local + @$(MAKE) HUGO_FLAGS='-DF' DOMAIN=staging.seirdy.one USER=deploy@seirdy.one OUTPUT_DIR=public_staging copy-to-xhtml @$(MAKE) HUGO_FLAGS='-DF' DOMAIN=staging.seirdy.one USER=deploy@seirdy.one OUTPUT_DIR=public_staging deploy-html .PHONY: deploy-envs deploy-envs: @$(MAKE) NO_STATIC=1 HUGO_FLAGS='' USER=seirdy@envs.net WWW_ROOT=/home/seirdy/public_html GEMINI_ROOT=/home/seirdy/public_gemini HUGO_BASEURL='https://envs.net/~seirdy/' OUTPUT_DIR=public_envs .prepare-deploy @$(MAKE) NO_STATIC=1 HUGO_FLAGS='' USER=seirdy@envs.net WWW_ROOT=/home/seirdy/public_html GEMINI_ROOT=/home/seirdy/public_gemini HUGO_BASEURL='https://envs.net/~seirdy/' OUTPUT_DIR=public_envs lint-local + @$(MAKE) NO_STATIC=1 HUGO_FLAGS='' USER=seirdy@envs.net WWW_ROOT=/home/seirdy/public_html GEMINI_ROOT=/home/seirdy/public_gemini HUGO_BASEURL='https://envs.net/~seirdy/' OUTPUT_DIR=public_envs copy-to-xhtml @$(MAKE) NO_STATIC=1 HUGO_FLAGS='' USER=seirdy@envs.net WWW_ROOT=/home/seirdy/public_html GEMINI_ROOT=/home/seirdy/public_gemini HUGO_BASEURL='https://envs.net/~seirdy/' OUTPUT_DIR=public_envs deploy diff --git a/Makefile.online b/Makefile.online index bed37e8..0eba318 100644 --- a/Makefile.online +++ b/Makefile.online @@ -36,7 +36,8 @@ axe: @axe $(URLS) --chrome-options $(CHROME_FLAGS_COMMA) --chromedriver-path=$(CHROMEDRIVER_PATH) --show-errors axe-ff: @echo "Running axe with Firefox" - PATH=scripts/bin:$(PATH) axe $(URLS) -b firefox --show-errors + @scripts/bin/axe-ff $(OUTPUT_DIR) $(URLS) + .PHONY: axe axe-ff .validate-feed-main: diff --git a/scripts/bin/axe-ff b/scripts/bin/axe-ff new file mode 100755 index 0000000..8c7d5b6 --- /dev/null +++ b/scripts/bin/axe-ff @@ -0,0 +1,8 @@ +#!/bin/sh +# Runs axe on every page of my sitemap +# first arg is output directory, after that comes urls from a sitemap. We replace the urls with equivalent local file paths. +PATH="scripts/bin:$PATH" +output_dir="$1" +shift +urls="$(echo "$*" | tr ' ' '\n' | sd '^https?://[^/]*' "file://$PWD/$output_dir" | sd '/$' '/index.html' | tr '\n' ' ')" +axe $urls -b firefox --show-errors diff --git a/scripts/compress.sh b/scripts/compress.sh index 21a27dc..4549cb9 100644 --- a/scripts/compress.sh +++ b/scripts/compress.sh @@ -10,7 +10,7 @@ set -e -u output_dir="$1" format="$2" -alias find_compressible='find "$output_dir" -type f \( -name "*.html" -o -name "*.txt" -o -name "*.xhtml" -o -name "*.xml" -o -name "*.webmanifest" -o -name "*.*.svg" \)' +alias find_compressible='find "$output_dir" -type f \( -name "*.html" -o -name "*.txt" -o -name "*.xml" -o -name "*.webmanifest" -o -name "*.*.svg" \)' if [ "$format" = "gzip" ]; then compress_level="$3" diff --git a/scripts/copy-file-to-xhtml.sh b/scripts/copy-file-to-xhtml.sh new file mode 100644 index 0000000..7a635a0 --- /dev/null +++ b/scripts/copy-file-to-xhtml.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Take a single polygot (X)HTML file and make a .xhtml copy of it. Do +# the same for static-compressed versions. + +set -e -u + +html_file="$1" +xhtml_file="${html_file%*.html}.xhtml" + +cp -p "$html_file" "$xhtml_file" +[ -f "$html_file.gz" ] && cp -p "$html_file.gz" "$xhtml_file.gz" +[ -f "$html_file.br" ] && cp -p "$html_file.br" "$xhtml_file.br" diff --git a/scripts/xhtmlize-single-file.sh b/scripts/xhtmlize-single-file.sh index 794e643..6b9865f 100644 --- a/scripts/xhtmlize-single-file.sh +++ b/scripts/xhtmlize-single-file.sh @@ -26,7 +26,7 @@ trap cleanup EXIT # delete the stylesheet from the html file; we'll re-insert it later. # Also remove two indentation levels -sed 7d "$html_file" | xmllint --format --encode UTF-8 --noent - | sd '^\t(?:\t)?' '' >"$tmp_file" +sed 7d "$html_file" | xmllint --format --encode UTF-8 --noent - | tail -n +2 | sd '^\t(?:\t)?' '' >"$tmp_file" { head -n7 "$tmp_file" | sd -s '/>' ' />' cat "$OUTPUT_DIR/tmp.css" @@ -39,12 +39,4 @@ sed 7d "$html_file" | xmllint --format --encode UTF-8 --noent - | sd '^\t(?:\t)? | sd '([a-z])<(data|time)' '$1 <$2' \ | sd '(]*rel="(?:nofollow ugc|ugc nofollow)"(?:[^>]*)?>liked)' ' $1' \ | sd -s '/>' ' />' -} >>"$xhtml_file" - -# replace the html file with the formatted xhtml5 file, excluding the -# XML declaration. -tail -n +2 "$xhtml_file" > "$html_file" - -# remove the redundant charset declaration from the xhtml file. It's the -# first thing in the . -sed -i 5d "$xhtml_file" # busybox sed supports "-i" +} >"$html_file"