Proxmox VE
High Availability Cluster

hostnamectl set-hostname pve01
hostname
# Expected output: pve01

hostnamectl set-hostname pve02   # on Node 2
hostnamectl set-hostname pve03   # on Node 3
hostname

127.0.0.1        localhost
192.168.10.11    pve01.cluster.local    pve01
192.168.10.12    pve02.cluster.local    pve02
192.168.10.13    pve03.cluster.local    pve03

# From pve01
ping -c 3 pve02
ping -c 3 pve03

# From pve02
ping -c 3 pve01
ping -c 3 pve03

# From pve03
ping -c 3 pve01
ping -c 3 pve02

apt update && apt full-upgrade -y
reboot

# List all interfaces
ip -o link show | awk '{print $2, $9}'

# Alternative
ls /sys/class/net/

auto lo
iface lo inet loopback

# Management Interface
auto eth0
iface eth0 inet static
    address 192.168.10.11/24
    gateway 192.168.10.1

# Corosync Heartbeat — dedicated NIC, no gateway
auto eth5
iface eth5 inet static
    address 192.168.20.11/24

# Migration Bond — 4 NICs in LACP
auto bond0
iface bond0 inet manual
    bond-slaves eth1 eth2 eth3 eth4
    bond-miimon 100
    bond-mode 802.3ad
    bond-xmit-hash-policy layer2+3
    bond-lacp-rate fast

# Migration Bridge over bond0
auto vmbr1
iface vmbr1 inet static
    address 192.168.30.11/24
    bridge-ports bond0
    bridge-stp off
    bridge-fd 0

# Apply network changes without rebooting
ifreload -a

# Verify bond is active and all slaves are up
cat /proc/net/bonding/bond0

# Verify all interfaces are up with correct IPs
ip addr show

# Test migration network connectivity (from pve01)
ping -c 3 192.168.30.12   # pve02 migration IP
ping -c 3 192.168.30.13   # pve03 migration IP

# Test Corosync network connectivity (from pve01)
ping -c 3 192.168.20.12
ping -c 3 192.168.20.13

pvecm create proxmox-ha-cluster --ring0_addr 192.168.20.11
pvecm status

pvecm add 192.168.10.11 \
    --ring0_addr 192.168.20.12 \
    --use_ssh

pvecm status

# Expected:
#   Name: proxmox-ha-cluster
#   Nodes: 3
#   Quorum: 2  (quorate)
#   Nodeid   Votes  Name
#   1        1      pve01 (local)
#   2        1      pve02
#   3        1      pve03

# Add PBS repository
echo 'deb http://download.proxmox.com/debian/pbs bookworm pbs-no-subscription' \
    > /etc/apt/sources.list.d/pbs.list

# Add Proxmox GPG key
wget https://enterprise.proxmox.com/debian/proxmox-release-bookworm.gpg \
    -O /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg

# Install & start
apt update && apt install -y proxmox-backup-server
systemctl enable --now proxmox-backup

# Create dedicated backup user
proxmox-backup-manager user create backup-user@pbs \
    --password 'StrongPassword123!'

# Grant datastore permissions
proxmox-backup-manager acl update /datastore/vm-backups \
    --auth-id 'backup-user@pbs' \
    --role DatastoreBackup

# Generate API token (save the output — won't be shown again)
proxmox-backup-manager user generate-token backup-user@pbs cluster-token

# Verify sdb is the correct disk — CHECK CAREFULLY
lsblk

# Wipe all signatures, partitions, filesystem headers
wipefs -af /dev/sdb
sgdisk --zap-all /dev/sdb

# Zero the first 100MB to clear any LVM/RAID metadata
dd if=/dev/zero of=/dev/sdb bs=1M count=100 status=progress

# Confirm disk is clean
lsblk /dev/sdb
# Expected: sdb  [no children, no filesystem]

pveceph install --repository no-subscription
ceph --version

pveceph init --network 192.168.10.0/24

# The network here defines where Ceph public traffic goes.
# For a dedicated Ceph network, use your storage network instead.

pveceph mon create     # on pve01, pve02, and pve03

# Verify
ceph mon stat
# Expected: e3: 3 mons at {...}, quorum 0,1,2

pveceph mgr create     # on each node
ceph mgr stat

pveceph osd create /dev/sdb     # on each node

# Verify all 3 OSDs are up and in
ceph osd stat
# Expected: 3 osds: 3 up, 3 in

ceph health
# Expected: HEALTH_OK

ceph status

pveceph pool create VM-Disk --size 3 --min_size 2 --pg_autoscale_mode on
pvesm add rbd VM-Disk --pool VM-Disk --content images --krbd 0

ceph osd pool ls
ceph osd pool stats VM-Disk

ceph health detail
ceph status

# Expected:
#   cluster:   HEALTH_OK
#   services:  mon: 3 daemons, quorum pve01,pve02,pve03
#              mgr: pve01(active), pve02, pve03
#              osd: 3 osds: 3 up, 3 in
#   data:      pools: 1 pools, N pgs · pgs: N active+clean

echo 'deb http://download.proxmox.com/debian/pdm bookworm pdm-no-subscription' \
    > /etc/apt/sources.list.d/pdm.list

wget https://enterprise.proxmox.com/debian/proxmox-release-bookworm.gpg \
    -O /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg

apt update && apt install -y proxmox-datacenter-manager
systemctl enable --now proxmox-datacenter-manager
systemctl status proxmox-datacenter-manager

systemctl status pve-ha-crm
systemctl status pve-ha-lrm
ha-manager status

ha-manager add vm:100 --group production-ha --state started
ha-manager status
ha-manager resources

pvecm status
pvecm nodes
corosync-cfgtool -s

# Expected ring output (Link ID 0 udp):
#   nodeid: 2:  connected
#   nodeid: 3:  connected

ceph status
ceph osd tree
ceph osd pool stats VM-Disk

# I/O benchmarks
rados bench -p VM-Disk 10 write --no-cleanup
rados bench -p VM-Disk 10 seq
rados bench -p VM-Disk 10 rand

# Clean up
rados -p VM-Disk cleanup

# Start a test VM on pve01 (VM ID 100 on the VM-Disk pool)
qm start 100

# Live migrate to pve02
qm migrate 100 pve02 --online 1
qm status 100

# Migrate back
qm migrate 100 pve01 --online 1

# 1. Ensure test VM 100 is running on pve01
ha-manager status

# 2. Monitor HA events from pve02
journalctl -u pve-ha-crm -f

# 3. Simulate failure (hard poweroff)
# Press the power button, or via IPMI:
#   ipmitool chassis power off
# OR less destructive — isolate Corosync only:
iptables -A INPUT  -p udp --dport 5405 -j DROP
iptables -A OUTPUT -p udp --dport 5405 -j DROP

# 4. Within ~60 seconds: pve01 is detected down; VM 100 restarts on pve02 or pve03

# 5. Verify on surviving node
qm status 100

# 6. Restore pve01 and verify rejoin
pvecm status

# Remove iptables rules if used
iptables -D INPUT  -p udp --dport 5405 -j DROP
iptables -D OUTPUT -p udp --dport 5405 -j DROP

# Trigger manual backup of VM 100 to PBS
vzdump 100 --storage PBS --mode snapshot --compress zstd

# List backups
proxmox-backup-client list

# Restore from PBS to a different VM ID (non-destructive)
qmrestore PBS:backup/vzdump-qemu-100-<timestamp>.vma.zst 999 \
    --storage VM-Disk

# Verify
qm start 999
qm status 999

# Clean up
qm destroy 999

qm list
pct list
ls /etc/pve/qemu-server/
ls /etc/pve/lxc/

# Backup configs
mkdir -p /root/vm-config-backup
cp /etc/pve/qemu-server/*.conf /root/vm-config-backup/ 2>/dev/null
cp /etc/pve/lxc/*.conf /root/vm-config-backup/ 2>/dev/null

# Remove to allow join
rm -f /etc/pve/qemu-server/*.conf
rm -f /etc/pve/lxc/*.conf

# Retry join, then restore configs
cp /root/vm-config-backup/*.conf /etc/pve/qemu-server/

# Temporarily allow cluster operation with 1 node (split-brain risk)
pvecm expected 1
corosync-quorumtool -s

corosync-cfgtool -s
systemctl restart corosync       # caution in production
journalctl -u corosync -n 100

ceph health detail
ceph pg stat
ceph osd tree | grep -i down
systemctl status ceph-osd@<OSD_ID>
systemctl start ceph-osd@<OSD_ID>

ceph df
ceph osd df

# Emergency only — temporarily raise full ratio
ceph osd set-nearfull-ratio 0.90
ceph osd set-full-ratio 0.95

ha-manager status
journalctl -u pve-ha-crm -n 50
journalctl -u pve-ha-lrm -n 50

# CRITICAL: verify VM disk is on shared Ceph storage
qm config <VMID> | grep scsi
# Must show VM-Disk: prefix, NOT local-lvm:

# Check bond status — all 4 slaves should be active
cat /proc/net/bonding/bond0

# Test migration bandwidth with iperf3
# On pve02 (receiver)
iperf3 -s -B 192.168.30.12

# On pve01 (sender)
iperf3 -c 192.168.30.12 -P 4 -t 30

# Confirm migration is using the right network
cat /etc/pve/datacenter.cfg | grep migration