Security Incident

INCIDENT=incident-$(date -u +%Y%m%dT%H%M%S)
EVIDENCE=/secure/${INCIDENT}/evidence
sudo mkdir -p "$EVIDENCE"
sudo chmod 0700 "$(dirname "$EVIDENCE")"

cat | sudo tee "$EVIDENCE/00-observations.txt" >/dev/null <<EOF
Incident: ${INCIDENT}
Started:  $(date -u +"%Y-%m-%dT%H:%M:%SZ")
Reporter: <name>
Detection: <how was it noticed>
Initial scope: <hosts, principals, time window>
Initial severity: Sev-<n>
EOF

# Stop the control plane. systemd drains in-flight gRPC streams
# within TimeoutStopSec=30s.
sudo systemctl stop kscore-server

# Block all but the operator's SSH source so the host is
# inspectable but unreachable to attacker traffic. Adjust to
# match your firewall (nftables / iptables / firewalld).
sudo iptables -I INPUT 1 -p tcp --dport 22 -s <operator-ip> -j ACCEPT
sudo iptables -A INPUT -j DROP

# On each suspect agent host:
sudo systemctl stop kscore-agent

# Optionally block its outbound NATS to ensure it can't reconnect
# until investigation is complete.
sudo iptables -A OUTPUT -p tcp --dport 4222 -j DROP

# Process tree.
sudo ps auxf > "$EVIDENCE/$(hostname)-processes.txt"

# Open sockets with owning PIDs.
sudo ss -tuanp > "$EVIDENCE/$(hostname)-sockets.txt"

# Open files (large; trim post-hoc).
sudo lsof -n > "$EVIDENCE/$(hostname)-lsof.txt"

# Logged-in users + recent logins.
who   > "$EVIDENCE/$(hostname)-who.txt"
last -20 > "$EVIDENCE/$(hostname)-last.txt"

# Recent journal lines for the kscore units (no truncation).
sudo journalctl -u kscore-server --since "24 hours ago" --no-pager \
    > "$EVIDENCE/$(hostname)-server.log" 2>/dev/null || true
sudo journalctl -u kscore-agent  --since "24 hours ago" --no-pager \
    > "$EVIDENCE/$(hostname)-agent.log"  2>/dev/null || true

# Full journal (auth, sudo, ssh attempts).
sudo journalctl --since "24 hours ago" --no-pager \
    > "$EVIDENCE/$(hostname)-journal.log"

# auditd events (if installed).
sudo ausearch -ts recent > "$EVIDENCE/$(hostname)-ausearch.txt" 2>/dev/null || true

# Config: server.yaml + agent.yaml as on-disk (REDACT BEFORE
# SHARING -- the HMAC secret + master key live here).
sudo cp /etc/kscore/server.yaml "$EVIDENCE/server.yaml.at-incident" 2>/dev/null || true
sudo cp /etc/kscore/agent.yaml  "$EVIDENCE/agent.yaml.at-incident"  2>/dev/null || true

# State DB (SQLite default). Copy with the server stopped so the
# WAL doesn't move under you.
sudo cp /var/lib/kscore/keystone.db    "$EVIDENCE/keystone.db.at-incident"    2>/dev/null || true
sudo cp /var/lib/kscore/keystone.db-wal "$EVIDENCE/keystone.db-wal.at-incident" 2>/dev/null || true
sudo cp /var/lib/kscore/keystone.db-shm "$EVIDENCE/keystone.db-shm.at-incident" 2>/dev/null || true

# JetStream embedded store (events + audit streams).
sudo tar czf "$EVIDENCE/jetstream.tar.gz" -C /var/lib/kscore jetstream 2>/dev/null || true

# Installed binaries (hash + mtime -- was kscore-server replaced?).
for bin in /usr/bin/kscore-server /usr/bin/kscore-agent /usr/bin/kscorectl \
           /usr/bin/kscore-audit /usr/bin/kscore-events /usr/bin/kscore-identity \
           /usr/bin/kscore-policy /usr/bin/kscore-secrets /usr/bin/kscore-cluster-backup; do
    [ -f "$bin" ] && sha256sum "$bin" >> "$EVIDENCE/binary-hashes.txt"
    [ -f "$bin" ] && ls -la "$bin"   >> "$EVIDENCE/binary-mtimes.txt"
done

# Installed package versions.
sudo dpkg -l kscore-server kscorectl kscore-agent > "$EVIDENCE/installed.txt" 2>/dev/null || \
    sudo rpm -q kscore-server kscorectl kscore-agent > "$EVIDENCE/installed.txt"

# SQLite row count snapshot.
sudo -u kscore sqlite3 /var/lib/kscore/keystone.db \
    "SELECT COUNT(*) FROM audit_events" > "$EVIDENCE/audit-rowcount-pre.txt"

# Export the last 7 days of audit entries to a sealed file. The
# export subcommand was added in Epic 12 task 15.
kscore-audit export \
    --format jsonl \
    --since 7d \
    --output-file "$EVIDENCE/audit-7d.jsonl"

sha256sum "$EVIDENCE/audit-7d.jsonl" > "$EVIDENCE/audit-7d.jsonl.sha256"

# All actions by a specific principal in the suspect window.
kscore-audit log \
    --user "<principal>" \
    --since "2026-05-24T00:00:00Z" \
    --until "2026-05-24T06:00:00Z" \
    -o json | tee "$EVIDENCE/audit-by-user.json"

# All actions matching an action pattern (e.g. anything that
# touched secrets).
kscore-audit log \
    --action "secret.read" \
    --since 24h \
    -o json | tee "$EVIDENCE/audit-secret-reads.json"

# High-severity events only.
kscore-audit log \
    --min-severity high \
    --since 7d \
    -o json | tee "$EVIDENCE/audit-high-severity.json"

# Tail-style live monitor while investigation runs.
watch -n 10 'kscore-audit log --limit 20'

# Construct an ordered timeline from the export.
cat "$EVIDENCE/audit-7d.jsonl" | \
    jq -s 'sort_by(.timestamp) | .[] |
           [.timestamp, .user, .action, .resource_type, .resource_id, .outcome] | @tsv' \
    > "$EVIDENCE/audit-timeline.tsv"

# Live tail.
kscore-events watch --since 5m

# Connection / disconnection patterns.
kscore-events list --type agent_connected    --since 24h
kscore-events list --type agent_disconnected --since 24h

# Command-exec events tied to a suspect agent.
kscore-events list --category job --since 24h -o json \
    | jq 'select(.source == "<agent-id>")'

# Severity-filtered replay over a wider window.
kscore-events replay \
    --since 7d \
    --min-severity error \
    -o json > "$EVIDENCE/events-7d-errors.json"

# CA inventory + current signing-CA fingerprint.
kscore-identity ca info -o json > "$EVIDENCE/ca-info.json"

# IdentityService runtime status (IdentityService gRPC IS
# registered in v0.1, unlike ClusterGRPCServer).
kscore-identity status -o json > "$EVIDENCE/identity-status.json"

# Outstanding bootstrap tokens. Unused tokens that should have
# been one-shot consumed are red flags.
kscore-identity token list -o json > "$EVIDENCE/tokens-all.json"
kscore-identity token list --unused    -o json > "$EVIDENCE/tokens-unused.json"
kscore-identity token list --unexpired -o json > "$EVIDENCE/tokens-unexpired.json"

# v0.1 exposes API key CRUD as REST under /api/v1/apikeys (mounted
# by pkg/api/server/handlers.go); there is no kscorectl apikey
# subcommand surface yet. Use the REST endpoint directly with the
# admin Bearer key. Substitute :8080 if you changed the HTTP port.

API_KEY=<your-admin-bearer-token>
curl -sS -H "Authorization: Bearer $API_KEY" \
    http://127.0.0.1:8080/api/v1/apikeys | jq > "$EVIDENCE/apikeys.json"

# Look for: keys you didn't create, keys created in the incident
# window, keys with role=admin you don't recognise.
jq '.apikeys[] | select(.created_at > "2026-05-24T00:00:00Z")' \
    "$EVIDENCE/apikeys.json"

# File-integrity check: were any kscore binaries replaced?
# Compare against a known-good hash database -- for v0.1 the
# release artifacts ship with sha256 sidecars in the same
# location as the .deb / .rpm.
sha256sum -c /tmp/known-good-hashes.txt

# Find files modified in the incident window.
sudo find /etc/kscore /var/lib/kscore /usr/bin/kscore-* \
    -newer "$EVIDENCE/00-observations.txt" -ls

# auditd records for the suspect window (if installed).
sudo ausearch -ts "06:00:00" -te "07:00:00" -m USER_CMD,EXECVE

# Established outbound connections (look for unexpected
# destinations).
sudo ss -tnp state established \
    | grep -vE ':(22|80|443|4222|5397|8080|8081)\b'

# Recent logins / sudo invocations.
sudo journalctl _COMM=sshd  --since "24 hours ago" | grep -i accepted
sudo journalctl _COMM=sudo  --since "24 hours ago" | grep -i COMMAND

# Get the key's id (NOT the cleartext -- the cleartext is gone
# the moment it was issued).
curl -sS -H "Authorization: Bearer $API_KEY" \
    http://127.0.0.1:8080/api/v1/apikeys | jq '.apikeys[] | {id, name, role}'

# Delete it. Auth chain (API key -> RBAC -> rate-limit) re-verifies
# on every request, so the next request from a deleted key fails
# closed without a server restart.
curl -sS -X DELETE \
    -H "Authorization: Bearer $API_KEY" \
    "http://127.0.0.1:8080/api/v1/apikeys/<id-to-revoke>"

# Verify by attempting a call with the revoked key -- expect 401.
curl -i -H "Authorization: Bearer <revoked-key>" \
    http://127.0.0.1:8080/api/v1/apikeys

# Revoke by token id.
kscore-identity token revoke <token-id>

# Cleanup the revoked / expired token registry (housekeeping).
kscore-identity token cleanup

# Rotates the signing CA. Existing identities issued by the
# previous signing CA become untrusted; agents must re-bootstrap
# under the new CA before they can reconnect. The root CA is
# NOT rotated by this command -- root rotation is operator-driven
# manual ceremony and is not in v0.1 scope.
kscore-identity ca rotate-signing

# Verify the new signing-CA fingerprint surfaced.
kscore-identity ca info -o json | jq '.signing_ca.fingerprint'

sudo cp /etc/kscore/server.yaml /etc/kscore/server.yaml.pre-rotation
HMAC=$(openssl rand -hex 32)
sudo sed -i "s|^\(\s*hmacsecret:\s*\"\).*\"|\1${HMAC}\"|" /etc/kscore/server.yaml
sudo systemctl restart kscore-server

# 1. List snapshots taken before the incident window.
kscore-cluster-backup list /var/backups/kscore/

# 2. Verify the snapshot integrity before restoring.
kscore-cluster-backup verify --input /var/backups/kscore/<snap>.tar

# 3. Stop the server.
sudo systemctl stop kscore-server

# 4. Restore. --force is required to overwrite a populated cluster.
sudo kscore-cluster-backup restore \
    --input /var/backups/kscore/<pre-incident>.tar \
    --force

# 5. Restart and verify.
sudo systemctl start kscore-server
sleep 35
curl -fsS http://127.0.0.1:8080/health/ready | jq '.ready, .components'

# On the rebuilt host, after package install:
sudo $EDITOR /etc/kscore/agent.yaml   # set agent.id + nats.urls
sudo systemctl start kscore-agent

# From the operator workstation: confirm the agent reconnected.
kscore-events list --type agent_connected --since 5m

D=/tmp/incident-bundle-$(date -u +%Y%m%dT%H%M%S)
mkdir -p "$D"

# All evidence collected during the incident.
sudo cp -r "$(dirname "$EVIDENCE")" "$D/evidence/"

# Redact known-sensitive material from configs (best-effort --
# INSPECT BEFORE SHARING).
for f in "$D"/evidence/*/server.yaml.at-incident "$D"/evidence/*/agent.yaml.at-incident; do
    [ -f "$f" ] || continue
    sudo sed -i 's/^\(\s*hmacsecret:\s*"\).*"/\1REDACTED"/' "$f"
    sudo sed -i 's/^\(\s*master_key:\s*\).*$/\1REDACTED/'    "$f"
done

# Timeline (TSV of audit events in time order).
cp "$EVIDENCE/audit-timeline.tsv" "$D/timeline.tsv" 2>/dev/null || true

# Versions.
sudo dpkg -l kscore-server kscorectl kscore-agent > "$D/installed.txt" 2>/dev/null || \
    sudo rpm -q kscore-server kscorectl kscore-agent > "$D/installed.txt"
/usr/bin/kscore-server --version >> "$D/installed.txt"

# Health snapshot post-recovery.
curl -fsS http://127.0.0.1:8080/health/ready > "$D/health-ready.json" 2>/dev/null || true

# Bundle.
sudo tar czf "${D}.tar.gz" -C /tmp "$(basename "$D")"
echo "Incident bundle: ${D}.tar.gz"

# Audit: bulk secret reads from a single principal.
kscore-audit log --action "secret.read" --since 24h -o json \
    | jq -r '.[] | .user' | sort | uniq -c | sort -rn | head -10

# Audit: any policy delete attempts (policy CRUD is Unimplemented
# in v0.1 -- successful policy deletes should NOT exist; any audit
# row claiming one is a forensic artifact worth investigating).
kscore-audit log --action "policy.delete" --since 30d

# Events: agent disconnects clustered in time (possible mass
# eviction or NATS broker disruption).
kscore-events list --type agent_disconnected --since 1h -o json \
    | jq -r '.[] | .timestamp' | cut -c1-16 | sort | uniq -c

# Filesystem: kscore binaries modified outside the package
# manager's purview.
sudo find /usr/bin -name 'kscore-*' -newer /var/lib/dpkg/info/kscore-server.md5sums \
    -ls 2>/dev/null

# Network: long-lived connections to non-standard ports.
sudo ss -tnp state established \
    | grep -vE ':(22|80|443|4222|5397|8080|8081|6060)\b'

# Auth: ssh accepted-publickey events in the suspect window.
sudo journalctl _COMM=sshd --since "24 hours ago" \
    | grep -i 'Accepted publickey'