New note: some site optimizations

This works particularly well on my Atom feeds with embedded XHTML.

Makefile

@@ -4,7 +4,6 @@ DEVSERVER_URL="http://localhost:1313/"
 
 DOMAIN = seirdy.one
 HUGO_BASEURL = "https://$(DOMAIN)/"
-HUGO_FLAGS = --gc --ignoreCache
 USER = deploy@$(DOMAIN)
 WWW_ROOT = /var/www/$(DOMAIN)
 GEMINI_ROOT = /srv/gemini/$(DOMAIN)
@@ -145,7 +144,7 @@ deploy: deploy-html deploy-gemini
 .PHONY: .prepare-deploy
 .prepare-deploy:
 	@$(MAKE) clean
-	@$(MAKE) hugo
+	@$(MAKE) HUGO_FLAGS='--gc --ignoreCache' hugo
 	@$(MAKE) xhtmlize
 
 # deploy steps need to happen one at a time
@@ -187,5 +186,5 @@ lint-and-deploy-staging:
 .PHONY: deploy-envs
 deploy-envs:
 	@$(MAKE) -j1 HUGO_FLAGS='' USER=seirdy@envs.net WWW_ROOT=/home/seirdy/public_html GEMINI_ROOT=/home/seirdy/public_gemini HUGO_BASEURL='https://envs.net/~seirdy/' OUTPUT_DIR=public_envs .prepare-deploy copy-to-xhtml
-	@$(MAKE) HUGO_FLAGS='' USER=seirdy@envs.net WWW_ROOT=/home/seirdy/public_html GEMINI_ROOT=/home/seirdy/public_gemini HUGO_BASEURL='https://envs.net/~seirdy/' OUTPUT_DIR=public_envs html-validate validate-html validate-json lint-css
+	@$(MAKE) HUGO_FLAGS='' USER=seirdy@envs.net WWW_ROOT=/home/seirdy/public_html GEMINI_ROOT=/home/seirdy/public_gemini HUGO_BASEURL='https://envs.net/~seirdy/' OUTPUT_DIR=public_envs html-validate validate-html validate-json
 	@$(MAKE) SSHFLAGS='-o KexAlgorithms=curve25519-sha256@libssh.org' HUGO_FLAGS='' USER=seirdy@envs.net WWW_ROOT=/home/seirdy/public_html GEMINI_ROOT=/home/seirdy/public_gemini HUGO_BASEURL='https://envs.net/~seirdy/' OUTPUT_DIR=public_envs deploy

content/notes/some-site-optimizations.md

@@ -0,0 +1,11 @@
+---
+title: "Some site optimizations"
+date: 2023-11-25T19:18:14-08:00
+---
+I continue to work on my site over break. Today I got the build time down to 3.5 seconds for the Hugo build and just under 3 seconds for post-processing and syntax-checking (mostly using `xmllint`). I'm not parallelizing the post-processing since my laptop just has two cores (no simultaneous multithreading) and CI has one.
+
+Seven seconds is still too slow for changes to feel instant, though. I can't just rebuild on save. It's especially slow for a site with only <200 pages.
+
+Hugo's profiler tells me that a lot of the time is spent searching for Webmentions that match the current page. Perhaps I should do that server-side ahead of time; right now, I download all my Webmentions as one big fat JSON file and load it into memory once. I should also move some post-processing back into Hugo.
+
+I also re-used `xmllint` to minify my feeds and check their syntax. The minification worked particularly well since my feeds contain raw, un-escaped XHTML. Some clients still fetch my full-text feeds without transparent compression.

layouts/_default/_markup/render-link.atom.xml

@@ -1,12 +1,12 @@
 {{- $parsedDest := urls.Parse (.Destination | absURL) -}}
-{{- if (strings.HasPrefix .Destination "#") -}}<!-- if it's a fragment -->
+{{- if (strings.HasPrefix .Destination "#") -}}
-	<a href="{{ .Destination }}"{{ with .Title}} title="{{ . }}"{{ end }}>{{ .Text | safeHTML }}</a>
+<a href="{{ .Destination }}"{{ with .Title}} title="{{ . }}"{{ end }}>{{ .Text | safeHTML }}</a>
-{{- else if not (strings.HasPrefix .Destination $parsedDest.Scheme) -}}<!-- if it's a relative url -->
+{{- else if not (strings.HasPrefix .Destination $parsedDest.Scheme) -}}
-	{{- $baseURL := trim .Page.Site.BaseURL "/" -}}
+{{- $baseURL := trim .Page.Site.BaseURL "/" -}}
-	{{- $fragment := "" -}}
+{{- $fragment := "" -}}
-	{{- with $parsedDest.Fragment -}}{{- $fragment = printf "#%s" . -}}{{- end -}}
+{{- with $parsedDest.Fragment -}}{{- $fragment = printf "#%s" . -}}{{- end -}}
-	{{- $href := printf "%s%s%s" $baseURL $parsedDest.Path $fragment | absURL -}}
+{{- $href := printf "%s%s%s" $baseURL $parsedDest.Path $fragment | absURL -}}
-	<a href="{{ $href }}"{{ with .Title}} title="{{ . }}"{{ end }}>{{ .Text | safeHTML }}</a>
+<a href="{{ $href }}"{{ with .Title}} title="{{ . }}"{{ end }}>{{ .Text | safeHTML }}</a>
 {{- else -}}
-	<a href="{{ .Destination | htmlEscape }}"{{ with .Title}} title="{{ . }}"{{ end }}>{{ .Text | safeHTML }}</a>
+<a href="{{ .Destination | htmlEscape }}"{{ with .Title}} title="{{ . }}"{{ end }}>{{ .Text | safeHTML }}</a>
 {{- end -}}

layouts/_default/_markup/render-link.rss.xml

@@ -1,12 +1,12 @@
 {{- $parsedDest := urls.Parse (.Destination | absURL) -}}
-{{- if (strings.HasPrefix .Destination "#") -}}<!-- if it's a fragment -->
+{{- if (strings.HasPrefix .Destination "#") -}}
-	<a href="{{ .Destination }}"{{ with .Title}} title="{{ . }}"{{ end }}>{{ .Text | safeHTML }}</a>
+<a href="{{ .Destination }}"{{ with .Title}} title="{{ . }}"{{ end }}>{{ .Text | safeHTML }}</a>
-{{- else if not (strings.HasPrefix .Destination $parsedDest.Scheme) -}}<!-- if it's a relative url -->
+{{- else if not (strings.HasPrefix .Destination $parsedDest.Scheme) -}}
-	{{- $baseURL := trim .Page.Site.BaseURL "/" -}}
+{{- $baseURL := trim .Page.Site.BaseURL "/" -}}
-	{{- $fragment := "" -}}
+{{- $fragment := "" -}}
-	{{- with $parsedDest.Fragment -}}{{- $fragment = printf "#%s" . -}}{{- end -}}
+{{- with $parsedDest.Fragment -}}{{- $fragment = printf "#%s" . -}}{{- end -}}
-	{{- $href := printf "%s%s%s" $baseURL $parsedDest.Path $fragment | absURL -}}
+{{- $href := printf "%s%s%s" $baseURL $parsedDest.Path $fragment | absURL -}}
-	<a href="{{ $href }}"{{ with .Title}} title="{{ . }}"{{ end }}>{{ .Text | safeHTML }}</a>
+<a href="{{ $href }}"{{ with .Title}} title="{{ . }}"{{ end }}>{{ .Text | safeHTML }}</a>
 {{- else -}}
-	<a href="{{ .Destination }}"{{ with .Title}} title="{{ . }}"{{ end }}>{{ .Text | safeHTML }}</a>
+<a href="{{ .Destination }}"{{ with .Title}} title="{{ . }}"{{ end }}>{{ .Text | safeHTML }}</a>
 {{- end -}}

layouts/_default/list.atom.xml

@@ -14,13 +14,15 @@
 <?xml version="1.0" encoding="utf-8" standalone="yes"?>
 <feed
 	xmlns="http://www.w3.org/2005/Atom"
+	xml:lang="{{ .Site.LanguageCode }}"
 	xmlns:activity="http://activitystrea.ms/spec/1.0/"
+	xmlns:admin="http://webns.net/mvcb/"
 	xmlns:media="http://search.yahoo.com/mrss/"
 	xmlns:ostatus="http://ostatus.org/schema/1.0"
 	xmlns:poco="http://portablecontacts.net/spec/1.0"
+	xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
 	xmlns:sy="http://purl.org/rss/1.0/modules/syndication/"
-	xmlns:thr="http://purl.org/syndication/thread/1.0"
+	xmlns:thr="http://purl.org/syndication/thread/1.0">
-	xml:lang="{{ .Site.LanguageCode }}">
 	<title>{{ $title }} on {{ .Site.Title }}</title>
 	<subtitle>
 		{{- if .Params.Description -}}
@@ -35,11 +37,12 @@
 	<sy:updateFrequency>2</sy:updateFrequency>
   <author>
 		<activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-		<poco:preferredUsername>Seirdy</poco:preferredUsername>
+		<poco:preferredUsername>{{ .Site.Author.name }}</poco:preferredUsername>
-		<poco:displayName>Seirdy</poco:displayName>
+		<poco:displayName>{{ .Site.Author.nick }}</poco:displayName>
-    <name>Seirdy</name>
+    <name>{{ .Site.Author.name }}</name>
 		<uri>https://seirdy.one/</uri>
   </author>
+	<admin:errorReportsTo rdf:resource="mailto:seirdy@seirdy.one" />
   <updated>{{ now.Format "2006-01-02T15:04:05-07:00" | safeHTML }}</updated>
   <id>{{ .Permalink }}</id>
   {{ range $pages -}}
@@ -48,7 +51,7 @@
 		<title>{{ .Title }}</title>
 		<link rel="alternate" type="text/html" href="{{ .Permalink }}" /><id>{{ .Permalink }}</id>
 		<published>{{ .Date.Format "2006-01-02T15:04:05-07:00" | safeHTML }}</published><updated>{{ .Lastmod.Format "2006-01-02T15:04:05-07:00" | safeHTML }}</updated>
-		<author><name>Seirdy</name><uri>https://seirdy.one/</uri></author>
+		<author><name>{{ .Site.Author.name }}</name><uri>https://seirdy.one/</uri></author>
 		<rights>CC-BY-SA 4.0</rights>
 		{{- with .Params.replyURI -}}
 		<thr:in-reply-to ref="{{ . }}" href="{{ . }}" />

layouts/_default/list.html

@@ -24,7 +24,7 @@
 					</a>
 				</h2>
 				{{- .Scratch.Set "shortMeta" true -}}
-				{{- partial "post-meta.html" . }}
+				{{- partial "post-meta/post-meta.html" . }}
 				{{- if eq .Section "notes" }}
 				<div class="e-content entry-content" itemprop="articleBody">
 					{{- if .Params.replyURI -}}

layouts/about/list.html

@@ -28,7 +28,7 @@
 				</p>
 				<div>
 				<p class="p-summary entry-summary" itemprop="description">{{ .Description }}</p>
-				<p>{{- partial "wordcount.html" . -}}</p>
+				<p>{{- partial "post-meta/wordcount.html" . -}}</p>
 				</div>
 			</article>
 		</li>

layouts/meta/list.html

@@ -28,7 +28,7 @@
 				</p>
 				<div>
 				<p class="p-summary entry-summary" itemprop="description">{{ .Description }}</p>
-				<p>{{- partial "wordcount.html" . -}}</p>
+				<p>{{- partial "post-meta/wordcount.html" . -}}</p>
 				</div>
 			</article>
 		</li>

layouts/partials/full-article.html

@@ -1,7 +1,7 @@
 <header>
 	<h1 itemprop="name headline" class="p-name entry-title" id="h1" tabindex="-1">{{ .Title }}</h1>
 	{{- if not .Params.disableMeta -}}
-	{{- partial "post-meta.html" . -}}
+	{{- partial "post-meta/post-meta.html" . -}}
 	{{- end -}}
 </header>
 <hr />

layouts/partials/interact.html

@@ -3,5 +3,5 @@
 <h2 id="interact" tabindex="-1">Interact</h2>
 <p>You can interact by <a href="#webmentions">sending webmentions</a> or by visiting a syndicated copy of this post.</p>
 {{- partial "syndication.html" . -}}
-{{- partial "webmentions.html" . -}}
+{{- partial "webmentions/webmentions.html" . -}}
 </footer>

layouts/partials/post-meta/post-meta.html

@@ -34,7 +34,7 @@
 			{{- end -}}
 			{{- if not $isNotes -}}
 {{ if $needsList }}<li>{{ else }}<br />{{ end }}
-			{{ partial "wordcount.html" . -}}
+			{{ partial "post-meta/wordcount.html" . -}}
 {{ if $needsList }}</li>{{ end }}
 			{{- end -}}
 {{- if $needsList }}

layouts/partials/posts.html

@@ -27,7 +27,7 @@
 
 				<div>
 				<p class="p-summary entry-summary" itemprop="description">{{ .Description }}</p>
-				<p>{{- partial "wordcount.html" . -}}</p>
+				<p>{{- partial "post-meta/wordcount.html" . -}}</p>
 				</div>
 			</article>
 		</li>

layouts/partials/webmentions/all-webmentions.html

@@ -0,0 +1 @@
+{{- return (index .Site.Data "webmentions").items -}}

layouts/partials/webmentions/rewrites.html

@@ -0,0 +1,5 @@
+{{- $rewritesDict := dict "" "" -}}
+{{- range $i, $r := (getCSV "," "/csv/rewrites.csv") -}}
+	{{- $rewritesDict = merge $rewritesDict (dict (index $r 0) (index $r 1)) -}}
+{{- end -}}
+{{- return $rewritesDict -}}

layouts/partials/webmentions/wbm-links.html

@@ -0,0 +1 @@
+{{- return (slice "https://si3t.ch/log/2021-04-18-entetes-floc.html" "https://xmpp.org/2021/02/newsletter-02-feburary/" "https://gurlic.com/technology/post/393626430212145157" "https://gurlic.com/technology/post/343249858599059461" "https://www.librepunk.club/@penryn/108411423190214816" "https://benign.town/@josias/108457015755310198" "http://www.tuxmachines.org/node/148146" "https://i.reddit.com/r/web_design/comments/k0dmpj/an_opinionated_list_of_best_practices_for_textual/gdmxy4u/" "https://bbbhltz.space/posts/thoughts-on-tech-feb2021/" "https://jorts.horse/@alice/108477866954580532" "https://brid.gy/comment/mastodon/@Seirdy@pleroma.envs.net/AK7FeQ4h2tUCKNwlXc/AK7GtGkE7JOVgm1Cgi" "https://fosstodon.org/@werwolf/108529382741681838" "https://mastodon.social/@WahbAllat/108986614624476982" "https://linuxrocks.online/@friend/109029028283860044" "https://fosstodon.org/@fullstackthaumaturge/108765040526523487" "https://inhji.de/notes/an-opinionated-list-of-best-practices-for-textual-websites" "https://ravidwivedi.in/whatsapp/" "https://hackers.town/@theruran/108440680870400884" "https://hackers.town/@theruran/108440475936938471" "https://mckinley.cc/twtxt/2022-may-aug.html#2022-06-25T16:06:07-07:00" "https://tarnkappe.info/lesetipps-bayern-it-sicherheit-db-app-trackt-neue-eu-datenbank/" "https://catcatnya.com/@kescher/109221687024062842" "https://catcatnya.com/@kescher/109221707054861018" "https://catcatnya.com/@kescher/109221721385520640" "https://catcatnya.com/@kescher/109221750082044200" "https://brid.gy/post/twitter/seirdy/1536747178877673475" "https://markesler.com/blog/website-design/" "https://catcatnya.com/@kescher/108601418196537980" "https://chaos.social/@n0toose/109035270210401105" "https://nicfab.it/en/posts/aware-digital-communication-respecting-privacy-and-the-apps-or-services-you-choose/" "https://haxf4rall.com/2022/09/23/a-collection-of-articles-about-hardening-linux/" "https://mastodon.randomroad.social/@dctrud/108680634691924661" "https://brid.gy/post/twitter/seirdy/1535891978969174016" "https://mastodon.technology/@codeberg/108403449317373462" "https://harveyr.net/posts/14kb/" "https://brid.gy/comment/mastodon/@Seirdy@pleroma.envs.net/ANUjukccjwEmivz3ia/ANUmmjSDUviUeCz42S" "https://forum.fedeproxy.eu/t/keep-platform-open-article/161" "https://forum.forgefriends.org/t/keep-platform-open-article/161" "https://collapsitarian.io/notice/AP5QssaOBF4HLmDf8q") -}}

layouts/partials/webmentions/webmentions.html

@@ -1,13 +1,6 @@
-{{- $wbmLinks := (slice "https://si3t.ch/log/2021-04-18-entetes-floc.html" "https://xmpp.org/2021/02/newsletter-02-feburary/" "https://gurlic.com/technology/post/393626430212145157" "https://gurlic.com/technology/post/343249858599059461" "https://www.librepunk.club/@penryn/108411423190214816" "https://benign.town/@josias/108457015755310198" "http://www.tuxmachines.org/node/148146" "https://i.reddit.com/r/web_design/comments/k0dmpj/an_opinionated_list_of_best_practices_for_textual/gdmxy4u/" "https://bbbhltz.space/posts/thoughts-on-tech-feb2021/" "https://jorts.horse/@alice/108477866954580532" "https://brid.gy/comment/mastodon/@Seirdy@pleroma.envs.net/AK7FeQ4h2tUCKNwlXc/AK7GtGkE7JOVgm1Cgi" "https://fosstodon.org/@werwolf/108529382741681838" "https://mastodon.social/@WahbAllat/108986614624476982" "https://linuxrocks.online/@friend/109029028283860044" "https://fosstodon.org/@fullstackthaumaturge/108765040526523487" "https://inhji.de/notes/an-opinionated-list-of-best-practices-for-textual-websites" "https://ravidwivedi.in/whatsapp/" "https://hackers.town/@theruran/108440680870400884" "https://hackers.town/@theruran/108440475936938471" "https://mckinley.cc/twtxt/2022-may-aug.html#2022-06-25T16:06:07-07:00" "https://tarnkappe.info/lesetipps-bayern-it-sicherheit-db-app-trackt-neue-eu-datenbank/" "https://catcatnya.com/@kescher/109221687024062842" "https://catcatnya.com/@kescher/109221707054861018" "https://catcatnya.com/@kescher/109221721385520640" "https://catcatnya.com/@kescher/109221750082044200" "https://brid.gy/post/twitter/seirdy/1536747178877673475" "https://markesler.com/blog/website-design/" "https://catcatnya.com/@kescher/108601418196537980" "https://chaos.social/@n0toose/109035270210401105" "https://nicfab.it/en/posts/aware-digital-communication-respecting-privacy-and-the-apps-or-services-you-choose/" "https://haxf4rall.com/2022/09/23/a-collection-of-articles-about-hardening-linux/" "https://mastodon.randomroad.social/@dctrud/108680634691924661" "https://brid.gy/post/twitter/seirdy/1535891978969174016" "https://mastodon.technology/@codeberg/108403449317373462" "https://harveyr.net/posts/14kb/" "https://brid.gy/comment/mastodon/@Seirdy@pleroma.envs.net/ANUjukccjwEmivz3ia/ANUmmjSDUviUeCz42S" "https://forum.fedeproxy.eu/t/keep-platform-open-article/161" "https://forum.forgefriends.org/t/keep-platform-open-article/161" "https://collapsitarian.io/notice/AP5QssaOBF4HLmDf8q") -}}
-{{- $rewritesDict := dict "" "" -}}
-{{- range $i, $r := (getCSV "," "/csv/rewrites.csv") -}}
-	{{- $rewritesDict = merge $rewritesDict (dict (index $r 0) (index $r 1)) -}}
-{{- end -}}
 <h3 id="webmentions" tabindex="-1">Web&#173;mentions</h3>
 <p>This site supports <a href="https://indieweb.org/Webmention">Webmentions</a>, a backlink-based alternative to traditional comment forms.</p>
-<details><summary>Send a Webmention</summary>{{ partial "webmention-form.html" . }}</details>
+<details><summary>Send a Webmention</summary>{{ partial "webmentions/webmention-form.html" . }}</details>
-{{- /* Fetch all webmentions with bearer token */ -}}
-{{- $allMentions := (index .Site.Data "webmentions").items -}}
 {{- /* Filter webmentions to just the ones for the current targets */ -}}
 	{{- $canonicalRelPermalink := .RelPermalink | replaceRE "^/~seirdy/" "/" -}}
 	{{- $target := printf "%s%s" .Site.Params.CanonicalBaseURL $canonicalRelPermalink -}}
@@ -19,7 +12,7 @@
 	{{- end -}}
 	{{- /* We can't just use a simple "where" function because we need to ignore URL anchors/query-params when making a comparison: https://discourse.gohugo.io/t/add-like-comparison-operator-to-the-where-function/42013/4 */ -}}
 	{{- $webmentions := slice -}}
-	{{- range $allMentions -}}
+	{{- range partialCached "webmentions/all-webmentions.html" . -}}
 		{{- if in $targets .target -}}
 			{{ $webmentions = $webmentions | append . }}
 		{{- else -}}
@@ -54,10 +47,10 @@
 				{{- $title = $webmention.title | strings.TrimSuffix "…" | truncate 128 -}}
 			{{- end -}}
 			{{- $src := $webmention.source -}}
-			{{- with index $rewritesDict $src -}}
+			{{- with index (partialCached "webmentions/rewrites.html" .) $src -}}
 				{{- $src = . -}}
 			{{- end -}}
-			{{- if in $wbmLinks $src -}}
+			{{- if in (partialCached "webmentions/wbm-links.html" .) $src -}}
 				{{- $src = printf "https://web.archive.org/web/0/%s" $src -}}
 			{{- end -}}
 			{{ if (eq $webmention.type "like") -}}

linter-configs/vnu_filter.jq

@@ -1,26 +1,27 @@
 # Filter false positives from the .messages entry of Nu Validator output
 .messages |= map(
 	. | select(
+		.type == "info" and (
+			.message == "Trailing slash on void elements has no effect and interacts badly with unquoted attribute values."
+			and (.url | test(".html"))
+		) or
 		.type == "error" and (
-			( # See https://github.com/w3c/css-validator/issues/361
-				.message == "CSS: Parse Error."
-				and .extract == "){outline:none}}@media(prefers"
-				and .hiliteLength == 1
-			)
-			or
 			( # See https://github.com/validator/validator/issues/1166
 			  # This false-positive has been fixed; will remove once validator.nu updates
 				# validator.w3.org/nu is up-to-date.
 				.message == "Attribute “media” not allowed on element “meta” at this point."
 				and (.extract | test(" name=\"theme-color\""))
-			)
+			) or
-			or
 			( # see https://github.com/w3c/css-validator/issues/369
 				.message == "CSS: “contain-intrinsic-size”: Property “contain-intrinsic-size” doesn't exist."
-			)
+			) or
-			or
 			( # see https://github.com/w3c/css-validator/issues/370
 				.message == "CSS: “contain”: “inline-size” is not a “contain” value."
+			) or
+			( # See https://github.com/w3c/css-validator/issues/361
+				.message == "CSS: Parse Error."
+				and .extract == "){outline:none}}@media(prefers"
+				and .hiliteLength == 1
 			)
 		) | not
 	)

scripts/compress.sh

@@ -5,6 +5,7 @@
 # 2. "gzip" or "brotli"
 # 3. ECT zopfli compression level
 
+# no pipefail here since there are no pipes.
 set -e -u
 
 output_dir="$1"

scripts/connectivity-check.sh

@@ -1,4 +1,8 @@
 #!/bin/sh
+# Ensure that we can connect to seirdy.one, and print the ipv4/ipv6 addresses used. The latter makes searching through my server logs easier.
+# Return a bad exit code if we can't connect over either ipv4 or ipv6.
+
+# no pipefail here since there are no pipes.
 set -e -u
 
 ipv6_success=1

scripts/copy-file-to-xhtml.sh

@@ -2,6 +2,7 @@
 # Take a single polygot (X)HTML file and make a .xhtml copy of it. Do
 # the same for static-compressed versions.
 
+# no pipefail here since there are no pipes.
 set -e -u
 
 html_file="$1"

scripts/curl-wrapper.sh

@@ -1,3 +1,6 @@
 #!/bin/sh
 # a curl wrapper for seirdy.one
-curl --proto "=https" --tlsv1.3 --cert-status -sS -m10 "$@"
+# no pipefail here since there are no pipes.
+
+set -e -u
+curl --proto "=https" --tlsv1.3 --cert-status -sS -m"${SEIRDYONE_CURL_TIMEOUT-10}" "$@"

scripts/filter-vnu.sh

@@ -1,13 +1,14 @@
 #!/bin/sh
 # filters the Nu HTML Validator's JSON output. Operates on stdin.
+# no pipefail here since there are no pipes.
 set -e -u
 
 root_dir="$(dirname "$0")/.."
 vnu_output="$(jaq --from-file "$root_dir/linter-configs/vnu_filter.jq")"
 if [ "$vnu_output" = '' ]; then
 	echo "All markup is valid"
-else
+	exit 0
-	echo "Markup errors found!" 1>&2
-	echo "$vnu_output"
-	exit 1
 fi
+echo "Markup errors found!" 1>&2
+echo "$vnu_output"
+exit 1

scripts/get-webmentions.sh

@@ -4,6 +4,7 @@
 # Uses POSIX and cURL in CI, also uses any pass/pash-compatible pwmngr otherwise
 # The response is cached for 90 minutes. If fetching fresh webmentions fails, fall back to the cached version for up to one day (1440 minutes). If cached webmentions are older than a day, error out.
 
+# no pipefail here since there are no pipes.
 set -e -u
 
 dirname="$(dirname "$0")"
@@ -32,7 +33,7 @@ check_cached_webmentions() {
 				echo 'Using cached webmentions'
 				skip_check=1
 				exit_status=0
-				return 0
+				exit 0
 			fi
 			return 1
 		fi
@@ -71,7 +72,7 @@ verify_webmentions() {
 # use the token to fetch all webmentions.
 fetch_webmentions() {
 	echo 'Fetching webmentions'
-	token_response="$(token)"
+	token_response="$(token)" || return 1
 	"$curl_wrapper" --compressed -H "Authorization: Bearer $token_response" "$webmentions_url" -o "$webmentions_file.tmp" || return 1
 	if verify_webmentions; then
 		mv "$webmentions_file.tmp" "$webmentions_file"

scripts/populate-webrings.sh

@@ -6,6 +6,9 @@
 # first-party HTML+CSS page. This script fetches webring links by
 # parsing the iframes and appends appropriate values to my webrings.csv file.
 
+#shellcheck disable=SC3040  # This only sets pipefail if it's available (e.g. in Bash, Zsh) and otherwise does nothing (in other POSIX shells like Busybox sh and dash)
+set -o pipefail 2>/dev/null
+
 set -e -u
 
 # the name of this program

scripts/setup-ci.sh

@@ -1,6 +1,8 @@
 #!/bin/sh
 # For CI: configures SSH and installs binaries that I use to build my site.
 # The binaries are statically-linked for Linux on x86_64
+
+# no pipefail here since there are no pipes.
 set -e -u
 
 # configure ssh

scripts/vnu.sh

@@ -1,16 +1,28 @@
 #!/bin/sh
+
+# Runs the Nu HTML checker (assumed to be available on your $PATH as `vnu`) on all pages.
+
+#shellcheck disable=SC3040  # This only sets pipefail if it's available and otherwise does nothing
+set -o pipefail 2>/dev/null || true
 set -e -u
 dirname="$(dirname "$0")"
 output_dir="$1"
 
+# List all HTML, XHTML, and SVG files we wish to check.
+# We test both XHTML and HTML files, despite the fact that they are identical, because we want to validate under both an HTML and XML parser. VNU has some HTML-only checks.
 find_files_to_analyze() {
-	find "$output_dir" -type f -name '*.xhtml' -o -name '*.svg' \
+	find "$output_dir" -type f -name '*.html' -o -name '*.xhtml' -o -name '*.svg' \
 		| grep -Ev '(bimi\.svg|search/index\.x?html)$'
 }
 
+# Force a non-zero exit code for vnu. We'll decide the exit status after filtering out false-posives.
+vnu_nofail() {
+	xargs vnu --stdout --format json --also-check-svg || true
+}
+
 # files_to_analyze="$(find_files_to_analyze)"
 
 # we skip the BIMI icon (VNU can't handle SVG 1.2) and the search page (it has raw templates).
 find_files_to_analyze \
-	| xargs vnu --stdout --format json --also-check-svg \
+	| vnu_nofail \
 	| sh "$dirname/filter-vnu.sh"

scripts/xhtmlize-single-file.sh

@@ -13,6 +13,8 @@
 # with sed.
 # It also decreases indents by one level
 
+#shellcheck disable=SC3040  # This only sets pipefail if it's available and otherwise does nothing
+set -o pipefail 2>/dev/null || true
 set -e -u
 
 html_file="$1"
@@ -28,7 +30,6 @@ run_tidy() {
 }
 
 # delete the stylesheet from the html file; we'll re-insert it later.
-# Also remove two indentation levels
 sed 7d "$html_file" | xmllint --format --encode UTF-8 --noent - | tail -n +2 >"$tmp_file"
 {
 	head -n7 "$tmp_file"
@@ -43,6 +44,5 @@ sed 7d "$html_file" | xmllint --format --encode UTF-8 --noent - | tail -n +2 >"$
 		-e 's|([a-z])<data|\1 <data|' \
 		-e 's#</span>(<a[^>]*rel="(nofollow ugc|ugc nofollow)"([^>]*)?>liked</a>)#</span> \1#' \
 		-e 's#^[\t\s]*<(code|/pre)#<\1#' \
-		"$tmp_file" \
+		"$tmp_file"
-		| awk '/^<\/code>/{printf "%s",$0;next}7'
 } >"$html_file"

scripts/xhtmlize.sh

@@ -3,6 +3,7 @@
 # Runs xhtmlize-single-file.sh on every single html file in the output dir.
 # exits if xhtmlize-single-file fails.
 
+# no pipefail here since there are no pipes.
 set -e -u
 
 output_dir="$1"
@@ -19,4 +20,5 @@ trap cleanup EXIT
 export XMLLINT_INDENT='	'
 export OUTPUT_DIR="$output_dir"
 find "$output_dir" -type f -name '*.html' -exec sh "$script_dir/xhtmlize-single-file.sh" {} \;
+find "$output_dir" -type f -name '*.xml' -exec xmllint --noblanks --encode UTF-8 --noent {} --output {} \;
 # done