257 lines
6.9 KiB
Bash
Executable file
257 lines
6.9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "${root}"
|
|
|
|
robot_base="${EVERY_CHANNEL_ROBOT_API_BASE:-https://robot-ws.your-server.de}"
|
|
server="${EVERY_CHANNEL_ROBOT_SERVER:-2800441}"
|
|
host_ip="${EVERY_CHANNEL_FORGE_IP:-95.216.114.54}"
|
|
host_name="${EVERY_CHANNEL_FORGE_HOSTNAME:-git.every.channel}"
|
|
op_item="${EVERY_CHANNEL_ROBOT_OP_ITEM:-Hetzner Robot}"
|
|
op_vault="${EVERY_CHANNEL_ROBOT_OP_VAULT:-}"
|
|
ssh_identity="${EVERY_CHANNEL_FORGE_SSH_IDENTITY:-$HOME/.ssh/id_ed25519}"
|
|
ssh_pub="${EVERY_CHANNEL_FORGE_SSH_PUBLIC_KEY:-${ssh_identity}.pub}"
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Usage: $0 <command>
|
|
|
|
Commands:
|
|
probe Check public HTTPS and SSH reachability for ${host_name}.
|
|
server Query Robot server metadata.
|
|
status Query Robot reset and rescue status.
|
|
rescue-status Query Robot rescue status.
|
|
activate-rescue Activate Linux rescue mode for ${server}.
|
|
reset [type] Execute a Robot reset, default: hw.
|
|
recover [type] Activate rescue mode, then execute a reset.
|
|
wait-ssh Wait until TCP/22 answers on ${host_ip}.
|
|
|
|
Credentials:
|
|
Set EVERY_CHANNEL_ROBOT_USER and EVERY_CHANNEL_ROBOT_PASSWORD, or sign in with
|
|
1Password CLI and keep the existing item named "${op_item}" available.
|
|
|
|
Optional:
|
|
EVERY_CHANNEL_ROBOT_OP_ITEM default: Hetzner Robot
|
|
EVERY_CHANNEL_ROBOT_OP_VAULT optional 1Password vault scope
|
|
EVERY_CHANNEL_ROBOT_AUTHORIZED_KEY_FINGERPRINT
|
|
EVERY_CHANNEL_ROBOT_PRINT_SENSITIVE=1 print Robot-generated rescue password
|
|
EVERY_CHANNEL_ROBOT_RESCUE_OS=linux
|
|
EVERY_CHANNEL_ROBOT_RESCUE_KEYBOARD=us
|
|
EVERY_CHANNEL_ROBOT_RESET_TYPE=hw
|
|
EOF
|
|
}
|
|
|
|
require_cmd() {
|
|
if ! command -v "$1" >/dev/null 2>&1; then
|
|
echo "error: required command not found: $1" >&2
|
|
exit 2
|
|
fi
|
|
}
|
|
|
|
op_field() {
|
|
local field="$1"
|
|
if [[ -n "${op_vault}" ]]; then
|
|
op item get "${op_item}" --vault "${op_vault}" --fields "label=${field}" --reveal
|
|
else
|
|
op item get "${op_item}" --fields "label=${field}" --reveal
|
|
fi
|
|
}
|
|
|
|
load_robot_auth() {
|
|
robot_user="${EVERY_CHANNEL_ROBOT_USER:-${HETZNER_ROBOT_USER:-}}"
|
|
robot_password="${EVERY_CHANNEL_ROBOT_PASSWORD:-${HETZNER_ROBOT_PASSWORD:-}}"
|
|
|
|
if [[ -z "${robot_user}" || -z "${robot_password}" ]]; then
|
|
require_cmd op
|
|
robot_user="${robot_user:-$(op_field username)}"
|
|
robot_password="${robot_password:-$(op_field password)}"
|
|
fi
|
|
|
|
if [[ -z "${robot_user}" || -z "${robot_password}" ]]; then
|
|
echo "error: Robot credentials are not available" >&2
|
|
echo "hint: run 'op signin' or export EVERY_CHANNEL_ROBOT_USER and EVERY_CHANNEL_ROBOT_PASSWORD" >&2
|
|
exit 2
|
|
fi
|
|
}
|
|
|
|
robot_curl() {
|
|
local method="$1"
|
|
local path="$2"
|
|
shift 2
|
|
|
|
local config
|
|
config="$(mktemp "${TMPDIR:-/tmp}/ec-robot-curl.XXXXXX")"
|
|
chmod 600 "${config}"
|
|
cleanup_config() {
|
|
rm -f "${config}"
|
|
}
|
|
trap cleanup_config RETURN
|
|
|
|
{
|
|
printf 'url = "%s%s"\n' "${robot_base}" "${path}"
|
|
printf 'request = "%s"\n' "${method}"
|
|
printf 'user = "%s:%s"\n' "${robot_user}" "${robot_password}"
|
|
printf 'silent\nshow-error\nfail\n'
|
|
} >"${config}"
|
|
|
|
curl --config "${config}" "$@"
|
|
}
|
|
|
|
mask_sensitive_json() {
|
|
if [[ "${EVERY_CHANNEL_ROBOT_PRINT_SENSITIVE:-0}" == "1" ]]; then
|
|
cat
|
|
return
|
|
fi
|
|
|
|
if command -v jq >/dev/null 2>&1; then
|
|
jq 'walk(if type == "object" and has("password") then .password = "<redacted>" else . end)'
|
|
else
|
|
sed -E 's/"password"[[:space:]]*:[[:space:]]*"[^"]*"/"password":"<redacted>"/g'
|
|
fi
|
|
}
|
|
|
|
authorized_key_fingerprint() {
|
|
if [[ -n "${EVERY_CHANNEL_ROBOT_AUTHORIZED_KEY_FINGERPRINT:-}" ]]; then
|
|
printf '%s\n' "${EVERY_CHANNEL_ROBOT_AUTHORIZED_KEY_FINGERPRINT}"
|
|
return
|
|
fi
|
|
|
|
if [[ -f "${ssh_pub}" ]] && command -v ssh-keygen >/dev/null 2>&1; then
|
|
ssh-keygen -E md5 -lf "${ssh_pub}" | awk '{print $2}' | sed 's/^MD5://'
|
|
fi
|
|
}
|
|
|
|
probe() {
|
|
echo "== HTTPS via DNS =="
|
|
curl --max-time 12 -I -sS "https://${host_name}/" || true
|
|
echo
|
|
echo "== HTTPS via pinned IP =="
|
|
curl --max-time 12 -I -sS --resolve "${host_name}:443:${host_ip}" "https://${host_name}/" || true
|
|
echo
|
|
echo "== SSH =="
|
|
ssh -o ConnectTimeout=12 -o BatchMode=yes -o IdentityAgent=none -o IdentitiesOnly=yes \
|
|
-i "${ssh_identity}" "root@${host_ip}" echo ssh-ok || true
|
|
}
|
|
|
|
server_metadata() {
|
|
load_robot_auth
|
|
robot_curl GET "/server/${server}" | mask_sensitive_json
|
|
}
|
|
|
|
reset_status() {
|
|
robot_curl GET "/reset/${server}" | mask_sensitive_json
|
|
}
|
|
|
|
rescue_status() {
|
|
load_robot_auth
|
|
robot_curl GET "/boot/${server}/rescue" | mask_sensitive_json
|
|
}
|
|
|
|
status() {
|
|
load_robot_auth
|
|
echo "== reset =="
|
|
reset_status
|
|
echo
|
|
echo "== rescue =="
|
|
robot_curl GET "/boot/${server}/rescue" | mask_sensitive_json
|
|
}
|
|
|
|
activate_rescue() {
|
|
load_robot_auth
|
|
local rescue_os="${EVERY_CHANNEL_ROBOT_RESCUE_OS:-linux}"
|
|
local keyboard="${EVERY_CHANNEL_ROBOT_RESCUE_KEYBOARD:-us}"
|
|
local key_fingerprint
|
|
key_fingerprint="$(authorized_key_fingerprint || true)"
|
|
|
|
local args=(--data-urlencode "os=${rescue_os}" --data-urlencode "keyboard=${keyboard}")
|
|
if [[ -n "${key_fingerprint}" ]]; then
|
|
args+=(--data-urlencode "authorized_key[]=${key_fingerprint}")
|
|
fi
|
|
|
|
robot_curl POST "/boot/${server}/rescue" "${args[@]}" | mask_sensitive_json
|
|
}
|
|
|
|
reset_server() {
|
|
load_robot_auth
|
|
local reset_type="${1:-${EVERY_CHANNEL_ROBOT_RESET_TYPE:-hw}}"
|
|
robot_curl POST "/reset/${server}" --data-urlencode "type=${reset_type}" | mask_sensitive_json
|
|
}
|
|
|
|
recover() {
|
|
local reset_type="${1:-${EVERY_CHANNEL_ROBOT_RESET_TYPE:-hw}}"
|
|
echo "== activate rescue =="
|
|
activate_rescue
|
|
echo
|
|
echo "== reset ${reset_type} =="
|
|
reset_server "${reset_type}"
|
|
echo
|
|
echo "Rescue boot requested. Run '$0 wait-ssh' to watch for SSH on ${host_ip}."
|
|
}
|
|
|
|
wait_ssh() {
|
|
local deadline="${EVERY_CHANNEL_FORGE_WAIT_SECONDS:-300}"
|
|
local started
|
|
started="$(date +%s)"
|
|
|
|
while true; do
|
|
if command -v nc >/dev/null 2>&1; then
|
|
if nc -z -w 5 "${host_ip}" 22 >/dev/null 2>&1; then
|
|
echo "ssh port is reachable on ${host_ip}:22"
|
|
return 0
|
|
fi
|
|
elif ssh -o ConnectTimeout=5 -o BatchMode=yes -o StrictHostKeyChecking=no \
|
|
-o UserKnownHostsFile=/dev/null "root@${host_ip}" true >/dev/null 2>&1; then
|
|
echo "ssh is reachable on ${host_ip}"
|
|
return 0
|
|
fi
|
|
|
|
if (( "$(date +%s)" - started >= deadline )); then
|
|
echo "error: timed out waiting for SSH on ${host_ip}:22" >&2
|
|
return 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
}
|
|
|
|
cmd="${1:-}"
|
|
case "${cmd}" in
|
|
""|-h|--help|help)
|
|
usage
|
|
;;
|
|
probe)
|
|
probe
|
|
;;
|
|
server)
|
|
require_cmd curl
|
|
server_metadata
|
|
;;
|
|
status)
|
|
require_cmd curl
|
|
status
|
|
;;
|
|
rescue-status)
|
|
require_cmd curl
|
|
rescue_status
|
|
;;
|
|
activate-rescue)
|
|
require_cmd curl
|
|
activate_rescue
|
|
;;
|
|
reset)
|
|
require_cmd curl
|
|
reset_server "${2:-}"
|
|
;;
|
|
recover)
|
|
require_cmd curl
|
|
recover "${2:-}"
|
|
;;
|
|
wait-ssh)
|
|
wait_ssh
|
|
;;
|
|
*)
|
|
echo "error: unknown command: ${cmd}" >&2
|
|
usage >&2
|
|
exit 2
|
|
;;
|
|
esac
|