diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml
index 6bbcdfebcce3..ac1f3c7b5777 100644
--- a/.github/workflows/labels.yml
+++ b/.github/workflows/labels.yml
@@ -137,7 +137,7 @@ jobs:
                     repositoryOwner: context.repo.owner,
                     token: core.getInput('github-token')
                   },
-                  path: path.resolve('comparison'),
+                  path: path.resolve(pull_request.number.toString()),
                   expectedHash: artifact.digest
                 })
 
@@ -161,11 +161,11 @@ jobs:
                 )
 
                 const maintainers = new Set(Object.keys(
-                  JSON.parse(await readFile('comparison/maintainers.json', 'utf-8'))
+                  JSON.parse(await readFile(`${pull_request.number}/maintainers.json`, 'utf-8'))
                 ))
 
                 // And the labels that should be there
-                const after = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8')).labels
+                const after = JSON.parse(await readFile(`${pull_request.number}/changed-paths.json`, 'utf-8')).labels
                 if (approvals.size > 0) after.push(`12.approvals: ${approvals.size > 2 ? '3+' : approvals.size}`)
                 if (Array.from(maintainers).some(m => approvals.has(m))) after.push('12.approved-by: package-maintainer')