From 8709bf9ec2b2e4ac51b4fbe18b8eab175455a827 Mon Sep 17 00:00:00 2001 From: Rohan Kumar Date: Mon, 19 Dec 2022 10:23:00 -0800 Subject: [PATCH] Overhaul webmention generation Use a new branch of webmentiond that lets me pull in all webmentions for all pages in a single JSON response Before, Hugo would make one request to webmentiond per page to ask for approved webmentions for that page. Sometimes, it makes two requests because some pages used to have a different canonical location. In all, it ended up making over 150 requests within a second or two. Webmentiond can handle this for now, but this isn't sustainable: page count will only increase with time. I wanted to have Hugo instead get all webmentions for all pages in one cached request. I recompiled webmentiond from https://github.com/zerok/webmentiond/pull/65, which updates the API to support admin access keys. The admin API allows pulling in all webmentions for all pages, instead of pulling them in for one page at a time. Doing so requires getting a bearer token, so I had to manage a new CI secret: the password for getting a token. I get the token in a shell script (get-token.sh) and write it to a temporary file, then have Hugo read the token from that file. The shell script gets the password using either the CI secret (in CI) or using my password manager (on my workstation). TODO: support marginalia (mentions with fragments in their targets) --- .build.yml | 3 ++- .gitignore | 1 + Makefile | 2 ++ layouts/partials/webmentions.html | 37 +++++++++++++++++++------------ scripts/get-token.sh | 32 ++++++++++++++++++++++++++ 5 files changed, 60 insertions(+), 15 deletions(-) create mode 100644 scripts/get-token.sh diff --git a/.build.yml b/.build.yml index c34e0a1..7176fb4 100644 --- a/.build.yml +++ b/.build.yml @@ -11,7 +11,8 @@ packages: sources: - https://git.sr.ht/~seirdy/seirdy.one secrets: - - cc1eb90c-b07b-4c46-86d4-58fec41cf0e4 + - cc1eb90c-b07b-4c46-86d4-58fec41cf0e4 # ssh key + - b0bfa66e-3cf2-468f-9ea4-085819eccd18 # webmentiond key triggers: - action: email condition: always diff --git a/.gitignore b/.gitignore index b4f80b3..b923906 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ public.eac # when i feel like it styles/ .vale.ini +.webmentiond-token diff --git a/Makefile b/Makefile index 233edd0..3a4c5b1 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,9 @@ csv/webrings.csv: .PHONY: hugo hugo: csv/webrings.csv $(SRCFILES) + sh scripts/get-token.sh hugo -b $(HUGO_BASEURL) $(HUGO_FLAGS) -d $(OUTPUT_DIR) + rm .webmentiond-token mv $(OUTPUT_DIR)/about/_index.gmi $(OUTPUT_DIR)/about/index.gmi # .hintrc-local for linting local files diff --git a/layouts/partials/webmentions.html b/layouts/partials/webmentions.html index 52bfce1..f8e94a1 100644 --- a/layouts/partials/webmentions.html +++ b/layouts/partials/webmentions.html @@ -1,5 +1,4 @@ {{- $wbmLinks := (slice "https://si3t.ch/log/2021-04-18-entetes-floc.html" "https://xmpp.org/2021/02/newsletter-02-feburary/" "https://gurlic.com/technology/post/393626430212145157" "https://gurlic.com/technology/post/343249858599059461" "https://www.librepunk.club/@penryn/108411423190214816" "https://benign.town/@josias/108457015755310198" "http://www.tuxmachines.org/node/148146" "https://i.reddit.com/r/web_design/comments/k0dmpj/an_opinionated_list_of_best_practices_for_textual/gdmxy4u/" "https://bbbhltz.space/posts/thoughts-on-tech-feb2021/" "https://jorts.horse/@alice/108477866954580532" "https://brid.gy/comment/mastodon/@Seirdy@pleroma.envs.net/AK7FeQ4h2tUCKNwlXc/AK7GtGkE7JOVgm1Cgi" "https://fosstodon.org/@werwolf/108529382741681838" "https://mastodon.social/@WahbAllat/108986614624476982" "https://linuxrocks.online/@friend/109029028283860044" "https://fosstodon.org/@fullstackthaumaturge/108765040526523487" "https://inhji.de/notes/an-opinionated-list-of-best-practices-for-textual-websites" "https://ravidwivedi.in/whatsapp/" "https://hackers.town/@theruran/108440680870400884" "https://hackers.town/@theruran/108440475936938471" "https://mckinley.cc/twtxt/2022-may-aug.html#2022-06-25T16:06:07-07:00" "https://tarnkappe.info/lesetipps-bayern-it-sicherheit-db-app-trackt-neue-eu-datenbank/" "https://catcatnya.com/@kescher/109221687024062842" "https://catcatnya.com/@kescher/109221707054861018" "https://catcatnya.com/@kescher/109221721385520640" "https://catcatnya.com/@kescher/109221750082044200" "https://brid.gy/post/twitter/seirdy/1536747178877673475" "https://markesler.com/blog/website-design/" "https://catcatnya.com/@kescher/108601418196537980" "https://chaos.social/@n0toose/109035270210401105" "https://nicfab.it/en/posts/aware-digital-communication-respecting-privacy-and-the-apps-or-services-you-choose/" "https://haxf4rall.com/2022/09/23/a-collection-of-articles-about-hardening-linux/" "https://mastodon.randomroad.social/@dctrud/108680634691924661" "https://brid.gy/post/twitter/seirdy/1535891978969174016" "https://mastodon.technology/@codeberg/108403449317373462" "https://harveyr.net/posts/14kb/" "https://brid.gy/comment/mastodon/@Seirdy@pleroma.envs.net/ANUjukccjwEmivz3ia/ANUmmjSDUviUeCz42S") -}} -{{- $archivetodayLinks := (slice "https://example.com") -}} {{- $rewritesDict := dict "" "" -}} {{- range $i, $r := (getCSV "," "/csv/rewrites.csv") -}} {{- $rewritesDict = merge $rewritesDict (dict (index $r 0) (index $r 1)) -}} @@ -7,21 +6,33 @@

Web­mentions

This site supports Webmentions, a backlink-based alternative to traditional comment forms.

{{ partial "webmention-form.html" . }} -{{- $target := .RelPermalink | replaceRE "^/~seirdy/" "/" }} -{{ $url1 := printf "https://seirdy.one/webmentions/get?status=approved&target=https://seirdy.one%s" $target -}} -{{ $webmentions := (getJSON $url1) -}} -{{- if lt .Date.Unix 1653616670 -}}{{- /* commit c84c8d4 changed my URL schemes, so fetch menchies for the legacy scheme on old posts. */ -}} - {{- $oldTarget := $target | replaceRE "/posts" "" | replaceRE "/$" ".html" -}} - {{- $url2 := printf "https://seirdy.one/webmentions/get?status=approved&target=https://seirdy.one%s" $oldTarget -}} - {{- $webmentions = $webmentions | append (getJSON $url2) -}} -{{- end -}} -{{- if gt (len $webmentions) 0 -}} +{{- /* Fetch all webmentions with bearer token */ -}} + {{ $bearer_token := "" }} + {{- with readFile ".webmentiond-token" -}} + {{- $auth := printf "Bearer %s" . -}} + {{- $bearer_token = dict "Authorization" $auth -}} + {{- end -}} + {{- $mentionsURL := "https://seirdy.one/webmentions/manage/mentions?limit=9999&status=approved" -}} + {{- $allMentions := (getJSON $mentionsURL $bearer_token).items -}} +{{- /* Filter webmentions to just the ones for the current targets */ -}} + {{- $canonicalRelPermalink := .RelPermalink | replaceRE "^/~seirdy/" "/" -}} + {{- $target := printf "%s%s" .Site.Params.CanonicalBaseURL $canonicalRelPermalink -}} + {{- $targets := (slice $target) -}} + {{- /* commit c84c8d4 changed my URL schemes, so fetch menchies for the legacy scheme on old posts. */ -}} + {{- if lt .Date.Unix 1653616670 -}} + {{- $oldTarget := $target | replaceRE "/posts" "" | replaceRE "/$" ".html" -}} + {{ $targets = (slice $target $oldTarget) }} + {{- end -}} + {{- $webmentions := where $allMentions "target" "in" $targets -}} +{{- /* Render webmentions if they exist */ -}} +{{- $count := (len $webmentions) -}} +{{- if gt $count 0 -}}

Webmentions received for this post appear in the following list after I approve them. I sometimes send Webmentions to myself on behalf of linking sites that don’t support them. I auto-replace broken links with Wayback Machine snapshots, if they exist.

- Toggle Webmentions + Toggle {{ $count }} Webmentions
{{ range sort $webmentions "created_at" -}} - {{ $webmention := . -}} + {{ $webmention := . -}} {{- /* Boolean: should we should handle this webmention like a comment or a linkback? */ -}} {{- $hasContent := and (isset $webmention "content") (gt (countrunes $webmention.content) 50) -}} {{- $title := $webmention.title -}} @@ -42,8 +53,6 @@ {{- end -}} {{- if in $wbmLinks $src -}} {{- $src = printf "https://web.archive.org/web/0/%s" $src -}} - {{- else if in $archivetodayLinks $src -}} - {{- $src = printf "https://archive.today/oldest/%s" $src -}} {{- end -}} {{ if (eq $webmention.type "like") -}}
diff --git a/scripts/get-token.sh b/scripts/get-token.sh new file mode 100644 index 0000000..ab06162 --- /dev/null +++ b/scripts/get-token.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Script to authenticate with webmentiond and grab a temporary generated +# bearer token, writing it to .webmentiond-token for Hugo to then read. + +set -e + +_key() { + if [ -n "$BUILD_SUBMITTER" ]; then + cat ~/.webmentiond-key + else + pash show webmentiond-ci-key + fi +} + +key="$(_key)" + +set -u + +# just a lil curl wrapper I use on seirdy.one +ccurl() { + curl --proto "=https" --proto-default https --tlsv1.3 --cert-status --compressed $* +} + +_token() { + ccurl -sX POST https://seirdy.one/webmentions/authenticate/access-key -d "key=$key" +} + +token="$(_token)" + +set +u +printf '%s' "$token" >.webmentiond-token