diff --git a/.github/workflows/keep-sorted.yml b/.github/workflows/keep-sorted.yml
index fbc39889b87b..c49b3c173b03 100644
--- a/.github/workflows/keep-sorted.yml
+++ b/.github/workflows/keep-sorted.yml
@@ -33,8 +33,8 @@ jobs:
           nix_path: nixpkgs=${{ env.url }}
 
       - name: Install keep-sorted
-        run: "nix-env -f '<nixpkgs>' -iAP keep-sorted"
+        run: "nix-env -f '<nixpkgs>' -iAP keep-sorted jq"
 
       - name: Check that Nix files are sorted
         run: |
-          git ls-files | xargs keep-sorted --mode lint
+          git ls-files | xargs keep-sorted --mode lint | jq --raw-output '.[] | "Please make sure any new entries in \(.path) are sorted alphabetically."'