diff --git a/tools/oncall/README.md b/tools/oncall/README.md
new file mode 100644
index 000000000000..5ada653c50c2
--- /dev/null
+++ b/tools/oncall/README.md
@@ -0,0 +1,8 @@
+# Oncall
+
+You can use the `oncall.sh` script to find out if there are any open issues or PRs from the community:
+```sh
+gh auth login
+./tools/oncall/oncall.sh issues
+./tools/oncall/oncall.sh prs
+```
diff --git a/scripts/on-call b/tools/oncall/oncall.sh
similarity index 78%
rename from scripts/on-call
rename to tools/oncall/oncall.sh
index 13cfdeed0bcc..7b8c3f1e0d80 100755
--- a/scripts/on-call
+++ b/tools/oncall/oncall.sh
@@ -9,7 +9,7 @@ usage() {
 Contains useful commands for on-call.
 
 Usage:
-    $(basename $0) <command>
+    $(basename "$0") <command>
 
 Commands:
     issues  List open issues from outside contributors.
@@ -18,7 +18,7 @@ EOF
 }
 
 require() {
-	type $1 >/dev/null 2>&1 || {
+	type "$1" >/dev/null 2>&1 || {
 		echo "$1 is required but not installed. Aborting." >&2
 		exit 1
 	}
@@ -29,12 +29,12 @@ issues() {
 	require jq
 
 	auth_status="$(gh auth status -t 2>&1)"
-	username="$(echo "${auth_status}" | sed -n -r 's/^.* Logged in to [^[:space:]]+ as ([^[:space:]]+).*/\1/p')"
+	username="$(echo "${auth_status}" | sed -n -r 's/^.* Logged in to github.com account ([^[:space:]]+).*/\1/p')"
 	token="$(echo "${auth_status}" | sed -n -r 's/^.*Token: ([a-zA-Z0-9_]*)/\1/p')"
 
-	members="$(curl -s -u "${username}:${token}" https://api.github.com/orgs/fleetdm/members | jq -r 'map(.login)')"
+	members="$(curl -s -u "${username}:${token}" https://api.github.com/orgs/fleetdm/members?per_page=100 | jq -r 'map(.login)')"
 
-	gh pr list --repo fleetdm/fleet --label "bug" --label ":reproduce" --json id,title,author,url,createdAt |
+	gh issue list --repo fleetdm/fleet --json id,title,author,url,createdAt,labels --limit 100 |
 		jq -r --argjson members "$members" \
 			'map(select(.author.login as $in | $members | index($in) | not)) | sort_by(.createdAt) | reverse'
 }
@@ -44,10 +44,11 @@ prs() {
 	require jq
 
 	auth_status="$(gh auth status -t 2>&1)"
-	username="$(echo "${auth_status}" | sed -n -r 's/^.* Logged in to [^[:space:]]+ as ([^[:space:]]+).*/\1/p')"
+	username="$(echo "${auth_status}" | sed -n -r 's/^.* Logged in to github.com account ([^[:space:]]+).*/\1/p')"
 	token="$(echo "${auth_status}" | sed -n -r 's/^.*Token: ([a-zA-Z0-9_]*)/\1/p')"
 
-	members="$(curl -s -u "${username}:${token}" https://api.github.com/orgs/fleetdm/members | jq -r 'map(.login)')"
+	members="$(curl -s -u "${username}:${token}" https://api.github.com/orgs/fleetdm/members?per_page=100 | jq -r 'map(.login)' | jq '. += ["app/dependabot"]')"
+
 	# defaults to listing open prs
 	gh pr list --repo fleetdm/fleet --json id,title,author,url,createdAt |
 		jq -r --argjson members "$members" \