Wire HDHomeRun observations and recover Forge OP Stack
This commit is contained in:
every.channel
parent
8065860449
commit
0d86104762
18 changed files with 1613 additions and 58 deletions
257
scripts/hetzner-robot-forge.sh
Executable file
257
scripts/hetzner-robot-forge.sh
Executable file
|
|
@ -0,0 +1,257 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "${root}"
|
||||
|
||||
robot_base="${EVERY_CHANNEL_ROBOT_API_BASE:-https://robot-ws.your-server.de}"
|
||||
server="${EVERY_CHANNEL_ROBOT_SERVER:-2800441}"
|
||||
host_ip="${EVERY_CHANNEL_FORGE_IP:-95.216.114.54}"
|
||||
host_name="${EVERY_CHANNEL_FORGE_HOSTNAME:-git.every.channel}"
|
||||
op_item="${EVERY_CHANNEL_ROBOT_OP_ITEM:-Hetzner Robot}"
|
||||
op_vault="${EVERY_CHANNEL_ROBOT_OP_VAULT:-}"
|
||||
ssh_identity="${EVERY_CHANNEL_FORGE_SSH_IDENTITY:-$HOME/.ssh/id_ed25519}"
|
||||
ssh_pub="${EVERY_CHANNEL_FORGE_SSH_PUBLIC_KEY:-${ssh_identity}.pub}"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 <command>
|
||||
|
||||
Commands:
|
||||
probe Check public HTTPS and SSH reachability for ${host_name}.
|
||||
server Query Robot server metadata.
|
||||
status Query Robot reset and rescue status.
|
||||
rescue-status Query Robot rescue status.
|
||||
activate-rescue Activate Linux rescue mode for ${server}.
|
||||
reset [type] Execute a Robot reset, default: hw.
|
||||
recover [type] Activate rescue mode, then execute a reset.
|
||||
wait-ssh Wait until TCP/22 answers on ${host_ip}.
|
||||
|
||||
Credentials:
|
||||
Set EVERY_CHANNEL_ROBOT_USER and EVERY_CHANNEL_ROBOT_PASSWORD, or sign in with
|
||||
1Password CLI and keep the existing item named "${op_item}" available.
|
||||
|
||||
Optional:
|
||||
EVERY_CHANNEL_ROBOT_OP_ITEM default: Hetzner Robot
|
||||
EVERY_CHANNEL_ROBOT_OP_VAULT optional 1Password vault scope
|
||||
EVERY_CHANNEL_ROBOT_AUTHORIZED_KEY_FINGERPRINT
|
||||
EVERY_CHANNEL_ROBOT_PRINT_SENSITIVE=1 print Robot-generated rescue password
|
||||
EVERY_CHANNEL_ROBOT_RESCUE_OS=linux
|
||||
EVERY_CHANNEL_ROBOT_RESCUE_KEYBOARD=us
|
||||
EVERY_CHANNEL_ROBOT_RESET_TYPE=hw
|
||||
EOF
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
echo "error: required command not found: $1" >&2
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
op_field() {
|
||||
local field="$1"
|
||||
if [[ -n "${op_vault}" ]]; then
|
||||
op item get "${op_item}" --vault "${op_vault}" --fields "label=${field}" --reveal
|
||||
else
|
||||
op item get "${op_item}" --fields "label=${field}" --reveal
|
||||
fi
|
||||
}
|
||||
|
||||
load_robot_auth() {
|
||||
robot_user="${EVERY_CHANNEL_ROBOT_USER:-${HETZNER_ROBOT_USER:-}}"
|
||||
robot_password="${EVERY_CHANNEL_ROBOT_PASSWORD:-${HETZNER_ROBOT_PASSWORD:-}}"
|
||||
|
||||
if [[ -z "${robot_user}" || -z "${robot_password}" ]]; then
|
||||
require_cmd op
|
||||
robot_user="${robot_user:-$(op_field username)}"
|
||||
robot_password="${robot_password:-$(op_field password)}"
|
||||
fi
|
||||
|
||||
if [[ -z "${robot_user}" || -z "${robot_password}" ]]; then
|
||||
echo "error: Robot credentials are not available" >&2
|
||||
echo "hint: run 'op signin' or export EVERY_CHANNEL_ROBOT_USER and EVERY_CHANNEL_ROBOT_PASSWORD" >&2
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
robot_curl() {
|
||||
local method="$1"
|
||||
local path="$2"
|
||||
shift 2
|
||||
|
||||
local config
|
||||
config="$(mktemp "${TMPDIR:-/tmp}/ec-robot-curl.XXXXXX")"
|
||||
chmod 600 "${config}"
|
||||
cleanup_config() {
|
||||
rm -f "${config}"
|
||||
}
|
||||
trap cleanup_config RETURN
|
||||
|
||||
{
|
||||
printf 'url = "%s%s"\n' "${robot_base}" "${path}"
|
||||
printf 'request = "%s"\n' "${method}"
|
||||
printf 'user = "%s:%s"\n' "${robot_user}" "${robot_password}"
|
||||
printf 'silent\nshow-error\nfail\n'
|
||||
} >"${config}"
|
||||
|
||||
curl --config "${config}" "$@"
|
||||
}
|
||||
|
||||
mask_sensitive_json() {
|
||||
if [[ "${EVERY_CHANNEL_ROBOT_PRINT_SENSITIVE:-0}" == "1" ]]; then
|
||||
cat
|
||||
return
|
||||
fi
|
||||
|
||||
if command -v jq >/dev/null 2>&1; then
|
||||
jq 'walk(if type == "object" and has("password") then .password = "<redacted>" else . end)'
|
||||
else
|
||||
sed -E 's/"password"[[:space:]]*:[[:space:]]*"[^"]*"/"password":"<redacted>"/g'
|
||||
fi
|
||||
}
|
||||
|
||||
authorized_key_fingerprint() {
|
||||
if [[ -n "${EVERY_CHANNEL_ROBOT_AUTHORIZED_KEY_FINGERPRINT:-}" ]]; then
|
||||
printf '%s\n' "${EVERY_CHANNEL_ROBOT_AUTHORIZED_KEY_FINGERPRINT}"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -f "${ssh_pub}" ]] && command -v ssh-keygen >/dev/null 2>&1; then
|
||||
ssh-keygen -E md5 -lf "${ssh_pub}" | awk '{print $2}' | sed 's/^MD5://'
|
||||
fi
|
||||
}
|
||||
|
||||
probe() {
|
||||
echo "== HTTPS via DNS =="
|
||||
curl --max-time 12 -I -sS "https://${host_name}/" || true
|
||||
echo
|
||||
echo "== HTTPS via pinned IP =="
|
||||
curl --max-time 12 -I -sS --resolve "${host_name}:443:${host_ip}" "https://${host_name}/" || true
|
||||
echo
|
||||
echo "== SSH =="
|
||||
ssh -o ConnectTimeout=12 -o BatchMode=yes -o IdentityAgent=none -o IdentitiesOnly=yes \
|
||||
-i "${ssh_identity}" "root@${host_ip}" echo ssh-ok || true
|
||||
}
|
||||
|
||||
server_metadata() {
|
||||
load_robot_auth
|
||||
robot_curl GET "/server/${server}" | mask_sensitive_json
|
||||
}
|
||||
|
||||
reset_status() {
|
||||
robot_curl GET "/reset/${server}" | mask_sensitive_json
|
||||
}
|
||||
|
||||
rescue_status() {
|
||||
load_robot_auth
|
||||
robot_curl GET "/boot/${server}/rescue" | mask_sensitive_json
|
||||
}
|
||||
|
||||
status() {
|
||||
load_robot_auth
|
||||
echo "== reset =="
|
||||
reset_status
|
||||
echo
|
||||
echo "== rescue =="
|
||||
robot_curl GET "/boot/${server}/rescue" | mask_sensitive_json
|
||||
}
|
||||
|
||||
activate_rescue() {
|
||||
load_robot_auth
|
||||
local rescue_os="${EVERY_CHANNEL_ROBOT_RESCUE_OS:-linux}"
|
||||
local keyboard="${EVERY_CHANNEL_ROBOT_RESCUE_KEYBOARD:-us}"
|
||||
local key_fingerprint
|
||||
key_fingerprint="$(authorized_key_fingerprint || true)"
|
||||
|
||||
local args=(--data-urlencode "os=${rescue_os}" --data-urlencode "keyboard=${keyboard}")
|
||||
if [[ -n "${key_fingerprint}" ]]; then
|
||||
args+=(--data-urlencode "authorized_key[]=${key_fingerprint}")
|
||||
fi
|
||||
|
||||
robot_curl POST "/boot/${server}/rescue" "${args[@]}" | mask_sensitive_json
|
||||
}
|
||||
|
||||
reset_server() {
|
||||
load_robot_auth
|
||||
local reset_type="${1:-${EVERY_CHANNEL_ROBOT_RESET_TYPE:-hw}}"
|
||||
robot_curl POST "/reset/${server}" --data-urlencode "type=${reset_type}" | mask_sensitive_json
|
||||
}
|
||||
|
||||
recover() {
|
||||
local reset_type="${1:-${EVERY_CHANNEL_ROBOT_RESET_TYPE:-hw}}"
|
||||
echo "== activate rescue =="
|
||||
activate_rescue
|
||||
echo
|
||||
echo "== reset ${reset_type} =="
|
||||
reset_server "${reset_type}"
|
||||
echo
|
||||
echo "Rescue boot requested. Run '$0 wait-ssh' to watch for SSH on ${host_ip}."
|
||||
}
|
||||
|
||||
wait_ssh() {
|
||||
local deadline="${EVERY_CHANNEL_FORGE_WAIT_SECONDS:-300}"
|
||||
local started
|
||||
started="$(date +%s)"
|
||||
|
||||
while true; do
|
||||
if command -v nc >/dev/null 2>&1; then
|
||||
if nc -z -w 5 "${host_ip}" 22 >/dev/null 2>&1; then
|
||||
echo "ssh port is reachable on ${host_ip}:22"
|
||||
return 0
|
||||
fi
|
||||
elif ssh -o ConnectTimeout=5 -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null "root@${host_ip}" true >/dev/null 2>&1; then
|
||||
echo "ssh is reachable on ${host_ip}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if (( "$(date +%s)" - started >= deadline )); then
|
||||
echo "error: timed out waiting for SSH on ${host_ip}:22" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
cmd="${1:-}"
|
||||
case "${cmd}" in
|
||||
""|-h|--help|help)
|
||||
usage
|
||||
;;
|
||||
probe)
|
||||
probe
|
||||
;;
|
||||
server)
|
||||
require_cmd curl
|
||||
server_metadata
|
||||
;;
|
||||
status)
|
||||
require_cmd curl
|
||||
status
|
||||
;;
|
||||
rescue-status)
|
||||
require_cmd curl
|
||||
rescue_status
|
||||
;;
|
||||
activate-rescue)
|
||||
require_cmd curl
|
||||
activate_rescue
|
||||
;;
|
||||
reset)
|
||||
require_cmd curl
|
||||
reset_server "${2:-}"
|
||||
;;
|
||||
recover)
|
||||
require_cmd curl
|
||||
recover "${2:-}"
|
||||
;;
|
||||
wait-ssh)
|
||||
wait_ssh
|
||||
;;
|
||||
*)
|
||||
echo "error: unknown command: ${cmd}" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
Loading…
Add table
Add a link
Reference in a new issue