Replace WireGuard with Nebula for true mesh networking

- Switch from WireGuard point-to-point to Nebula overlay network
- Certificate-based trust with single CA for cluster authentication
- True mesh networking - all nodes can communicate directly
- Simplified joining process with lighthouse-based discovery
- Network range: 192.168.100.0/24 (lighthouse at .1)
- Auto-downloads and installs Nebula binaries
- Maintains GlusterFS replication across mesh nodes

Note: Certificate distribution requires manual step for security
This commit is contained in:
Dominik Moritz Roth 2025-08-24 18:07:57 +02:00
parent 402d997599
commit 2c7bd4ac76
2 changed files with 254 additions and 128 deletions

View File

@ -45,7 +45,7 @@ The installer will:
## Nullpoint Cluster
Create or join a distributed storage cluster with WireGuard mesh networking and GlusterFS. Start with a single node and scale up by adding more servers.
Create or join a distributed storage cluster with Nebula mesh networking and GlusterFS. Start with a single node and scale up by adding more servers.
```bash
wget -qO- https://git.dominik-roth.eu/dodox/nullpoint/raw/branch/master/cluster-setup.sh | sudo bash
@ -53,5 +53,5 @@ wget -qO- https://git.dominik-roth.eu/dodox/nullpoint/raw/branch/master/cluster-
- **Storage mounted at**: `/data/storage/`
- **All data replicated** to all cluster nodes
- **Secure WireGuard mesh** between nodes
- **Secure Nebula mesh** - encrypted overlay network with certificate-based trust
- **Interactive setup** - choose create or join cluster

View File

@ -8,12 +8,13 @@ YELLOW='\033[1;33m'
NC='\033[0m' # No Color
# Configuration
WG_INTERFACE="wg-cluster"
WG_PORT=51820
WG_NETWORK="10.10.0.0/24"
NEBULA_NETWORK="192.168.100.0/24"
NEBULA_PORT=4242
NEBULA_CONFIG="/etc/nebula"
GLUSTER_BRICK_PATH="/gluster/cluster"
GLUSTER_MOUNT_PATH="/data/storage"
GLUSTER_VOLUME="cluster-volume"
NEBULA_VERSION="v1.8.2"
echo -e "${GREEN}================================${NC}"
echo -e "${GREEN} Nullpoint Cluster Setup${NC}"
@ -27,7 +28,15 @@ fi
# Install required packages
echo -e "${YELLOW}[+] Installing required packages...${NC}"
dnf install -y wireguard-tools glusterfs-server glusterfs-client || exit 1
dnf install -y curl tar glusterfs-server glusterfs-client || exit 1
# Download and install Nebula
echo -e "${YELLOW}[+] Downloading Nebula ${NEBULA_VERSION}...${NC}"
cd /tmp
curl -LO "https://github.com/slackhq/nebula/releases/download/${NEBULA_VERSION}/nebula-linux-amd64.tar.gz"
tar -zxf nebula-linux-amd64.tar.gz
mv nebula nebula-cert /usr/local/bin/
chmod +x /usr/local/bin/nebula /usr/local/bin/nebula-cert
# Enable and start GlusterFS
systemctl enable glusterd
@ -38,23 +47,35 @@ echo -e "${YELLOW}[+] Creating directories...${NC}"
mkdir -p "$GLUSTER_BRICK_PATH"
mkdir -p "$GLUSTER_MOUNT_PATH"
mkdir -p /data
mkdir -p "$NEBULA_CONFIG"
# Function to generate WireGuard keys
generate_wg_keys() {
local private_key=$(wg genkey)
local public_key=$(echo "$private_key" | wg pubkey)
local preshared_key=$(wg genpsk)
echo "$private_key:$public_key:$preshared_key"
# Function to generate Nebula CA and certificates
generate_nebula_ca() {
echo -e "${YELLOW}[+] Generating Nebula CA...${NC}"
cd "$NEBULA_CONFIG"
/usr/local/bin/nebula-cert ca -name "Nullpoint Cluster CA"
chmod 600 ca.key
}
# Function to create host certificate
create_host_cert() {
local hostname="$1"
local ip="$2"
local groups="$3"
cd "$NEBULA_CONFIG"
/usr/local/bin/nebula-cert sign -name "$hostname" -ip "$ip" -groups "$groups"
chmod 600 "${hostname}.key"
}
# Function to get next available IP
get_next_ip() {
local base_ip="10.10.0"
local next_num=2
local base_ip="192.168.100"
local next_num=10
if [ -f /etc/wireguard/${WG_INTERFACE}.conf ]; then
if [ -f "$NEBULA_CONFIG/cluster-registry.txt" ]; then
# Find highest IP in use
existing_ips=$(grep -E "AllowedIPs|Address" /etc/wireguard/${WG_INTERFACE}.conf | grep -oE "10\.10\.0\.[0-9]+" | cut -d. -f4 | sort -n | tail -1)
existing_ips=$(grep -oE "192\.168\.100\.[0-9]+" "$NEBULA_CONFIG/cluster-registry.txt" | cut -d. -f4 | sort -n | tail -1)
if [ ! -z "$existing_ips" ]; then
next_num=$((existing_ips + 1))
fi
@ -67,81 +88,146 @@ get_next_ip() {
setup_firewall() {
echo -e "${YELLOW}[+] Configuring firewall...${NC}"
# WireGuard
firewall-cmd --permanent --add-port=${WG_PORT}/udp
# Nebula
firewall-cmd --permanent --add-port=${NEBULA_PORT}/udp
# GlusterFS ports
firewall-cmd --permanent --add-service=glusterfs
firewall-cmd --permanent --add-port=24007-24008/tcp # GlusterFS Daemon
firewall-cmd --permanent --add-port=49152-49200/tcp # Brick ports
# Allow traffic from WireGuard network
firewall-cmd --permanent --zone=trusted --add-source=${WG_NETWORK}
# Allow traffic from Nebula network
firewall-cmd --permanent --zone=trusted --add-source=${NEBULA_NETWORK}
firewall-cmd --reload
}
# Create Nebula systemd service
create_nebula_service() {
cat > /etc/systemd/system/nebula.service <<EOF
[Unit]
Description=Nebula overlay networking tool
After=network.target
[Service]
Type=simple
User=root
Group=root
ExecStart=/usr/local/bin/nebula -config ${NEBULA_CONFIG}/config.yaml
ExecReload=/bin/kill -HUP \$MAINPID
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
}
# Create new cluster
create_cluster() {
echo -e "${GREEN}[*] Creating new cluster...${NC}\n"
# Generate keys
keys=$(generate_wg_keys)
private_key=$(echo "$keys" | cut -d: -f1)
public_key=$(echo "$keys" | cut -d: -f2)
preshared_key=$(echo "$keys" | cut -d: -f3)
local hostname=$(hostname)
local lighthouse_ip="192.168.100.1"
local external_ip=$(ip route get 1.1.1.1 | awk '{print $7; exit}')
# Create WireGuard config
cat > /etc/wireguard/${WG_INTERFACE}.conf <<EOF
[Interface]
Address = 10.10.0.1/24
ListenPort = ${WG_PORT}
PrivateKey = ${private_key}
SaveConfig = false
# Generate Nebula CA
generate_nebula_ca
# Create lighthouse certificate
create_host_cert "lighthouse" "${lighthouse_ip}/24" "lighthouse,cluster"
# Create Nebula config for lighthouse
cat > "${NEBULA_CONFIG}/config.yaml" <<EOF
pki:
ca: ${NEBULA_CONFIG}/ca.crt
cert: ${NEBULA_CONFIG}/lighthouse.crt
key: ${NEBULA_CONFIG}/lighthouse.key
# Peers will be added here
static_host_map:
"${lighthouse_ip}": ["${external_ip}:${NEBULA_PORT}"]
lighthouse:
am_lighthouse: true
serve_dns: false
interval: 60
hosts:
- "${lighthouse_ip}"
listen:
host: 0.0.0.0
port: ${NEBULA_PORT}
punchy:
punch: true
respond: true
tun:
disabled: false
dev: nebula1
drop_local_broadcast: false
drop_multicast: false
tx_queue: 500
mtu: 1300
logging:
level: info
format: text
firewall:
conntrack:
tcp_timeout: 12m
udp_timeout: 3m
default_timeout: 10m
max_connections: 100000
outbound:
- port: any
proto: any
host: any
inbound:
- port: any
proto: icmp
host: any
- port: any
proto: any
host: any
EOF
chmod 600 /etc/wireguard/${WG_INTERFACE}.conf
# Start WireGuard
systemctl enable wg-quick@${WG_INTERFACE}
systemctl start wg-quick@${WG_INTERFACE}
# Start Nebula as systemd service
create_nebula_service
systemctl enable nebula
systemctl start nebula
# Setup firewall
setup_firewall
# Create GlusterFS volume (single brick for now)
# Create cluster registry
echo "${lighthouse_ip} lighthouse ${hostname}" > "${NEBULA_CONFIG}/cluster-registry.txt"
# Create GlusterFS volume
echo -e "${YELLOW}[+] Creating GlusterFS volume...${NC}"
# Create brick directory
mkdir -p "${GLUSTER_BRICK_PATH}/brick1"
# Create volume with single brick (will be converted to replica when nodes join)
gluster volume create ${GLUSTER_VOLUME} $(hostname):${GLUSTER_BRICK_PATH}/brick1 force 2>/dev/null || true
gluster volume start ${GLUSTER_VOLUME} 2>/dev/null || true
# Mount the volume locally
# Mount volume
mount -t glusterfs localhost:/${GLUSTER_VOLUME} ${GLUSTER_MOUNT_PATH}
# Add to fstab for persistence
grep -q "${GLUSTER_VOLUME}" /etc/fstab || echo "localhost:/${GLUSTER_VOLUME} ${GLUSTER_MOUNT_PATH} glusterfs defaults,_netdev 0 0" >> /etc/fstab
# Get external IP for other nodes to connect
external_ip=$(ip route get 1.1.1.1 | awk '{print $7; exit}')
# Package CA certificate for sharing
ca_cert_b64=$(base64 -w0 "${NEBULA_CONFIG}/ca.crt")
echo -e "\n${GREEN}════════════════════════════════════════${NC}"
echo -e "${GREEN}Cluster created successfully!${NC}"
echo -e "${GREEN}════════════════════════════════════════${NC}\n"
echo -e "Share these details with nodes joining the cluster:\n"
echo -e "${YELLOW}Cluster IP:${NC} ${external_ip}"
echo -e "${YELLOW}WireGuard Port:${NC} ${WG_PORT}"
echo -e "${YELLOW}Public Key:${NC} ${public_key}"
echo -e "${YELLOW}Preshared Key:${NC} ${preshared_key}"
echo -e "\n${YELLOW}Cluster Secret (for easy sharing):${NC}"
echo -e "${GREEN}${external_ip}:${WG_PORT}:${public_key}:${preshared_key}${NC}"
echo -e "\n${YELLOW}Status:${NC}"
echo " - WireGuard interface: ${WG_INTERFACE} (10.10.0.1)"
echo -e "Share this cluster secret with joining nodes:\n"
echo -e "${GREEN}${external_ip}:${NEBULA_PORT}:${ca_cert_b64}${NC}\n"
echo -e "${YELLOW}Status:${NC}"
echo " - Nebula lighthouse: ${lighthouse_ip} (${external_ip}:${NEBULA_PORT})"
echo " - GlusterFS volume: ${GLUSTER_VOLUME}"
echo " - Mount point: ${GLUSTER_MOUNT_PATH}"
}
@ -150,92 +236,120 @@ EOF
join_cluster() {
echo -e "${GREEN}[*] Joining existing cluster...${NC}\n"
local hostname=$(hostname)
local my_ip=$(get_next_ip)
# Get cluster details
read -p "Enter cluster node IP: " cluster_ip
read -p "Enter cluster secret (or press enter to input separately): " cluster_secret
read -p "Enter cluster secret (lighthouse_ip:port:ca_cert_base64): " cluster_secret
if [ -z "$cluster_secret" ]; then
read -p "Enter WireGuard port [${WG_PORT}]: " wg_port_input
wg_port=${wg_port_input:-$WG_PORT}
read -p "Enter cluster public key: " cluster_public_key
read -p "Enter preshared key: " preshared_key
else
# Parse secret
cluster_ip=$(echo "$cluster_secret" | cut -d: -f1)
wg_port=$(echo "$cluster_secret" | cut -d: -f2)
cluster_public_key=$(echo "$cluster_secret" | cut -d: -f3)
preshared_key=$(echo "$cluster_secret" | cut -d: -f4)
echo -e "${RED}Cluster secret required!${NC}"
exit 1
fi
# Generate local keys
keys=$(generate_wg_keys)
private_key=$(echo "$keys" | cut -d: -f1)
public_key=$(echo "$keys" | cut -d: -f2)
# Parse secret
lighthouse_ip=$(echo "$cluster_secret" | cut -d: -f1)
nebula_port=$(echo "$cluster_secret" | cut -d: -f2)
ca_cert_b64=$(echo "$cluster_secret" | cut -d: -f3-)
# Get next available IP
my_ip=$(get_next_ip)
echo -e "${YELLOW}[+] Configuring Nebula (IP: ${my_ip})...${NC}"
echo -e "${YELLOW}[+] Configuring WireGuard (IP: ${my_ip})...${NC}"
# Decode and save CA certificate
echo "$ca_cert_b64" | base64 -d > "${NEBULA_CONFIG}/ca.crt"
# Create WireGuard config
cat > /etc/wireguard/${WG_INTERFACE}.conf <<EOF
[Interface]
Address = ${my_ip}/24
ListenPort = ${WG_PORT}
PrivateKey = ${private_key}
SaveConfig = false
echo -e "${RED}WARNING: Certificate signing not implemented in this simplified version.${NC}"
echo -e "${YELLOW}On the lighthouse node, run this command to create a certificate for this node:${NC}"
echo -e "${GREEN}cd ${NEBULA_CONFIG} && /usr/local/bin/nebula-cert sign -name \"${hostname}\" -ip \"${my_ip}/24\" -groups \"cluster\"${NC}"
echo -e "${YELLOW}Then copy ${hostname}.crt and ${hostname}.key to ${NEBULA_CONFIG}/ on this node.${NC}"
read -p "Press enter once you've created and copied the certificate files..."
# Create Nebula config
cat > "${NEBULA_CONFIG}/config.yaml" <<EOF
pki:
ca: ${NEBULA_CONFIG}/ca.crt
cert: ${NEBULA_CONFIG}/${hostname}.crt
key: ${NEBULA_CONFIG}/${hostname}.key
[Peer]
PublicKey = ${cluster_public_key}
PresharedKey = ${preshared_key}
Endpoint = ${cluster_ip}:${wg_port}
AllowedIPs = ${WG_NETWORK}
PersistentKeepalive = 25
static_host_map:
"192.168.100.1": ["${lighthouse_ip}:${nebula_port}"]
lighthouse:
am_lighthouse: false
interval: 60
hosts:
- "192.168.100.1"
listen:
host: 0.0.0.0
port: ${NEBULA_PORT}
punchy:
punch: true
respond: true
tun:
disabled: false
dev: nebula1
drop_local_broadcast: false
drop_multicast: false
tx_queue: 500
mtu: 1300
logging:
level: info
format: text
firewall:
conntrack:
tcp_timeout: 12m
udp_timeout: 3m
default_timeout: 10m
max_connections: 100000
outbound:
- port: any
proto: any
host: any
inbound:
- port: any
proto: icmp
host: any
- port: any
proto: any
host: any
EOF
chmod 600 /etc/wireguard/${WG_INTERFACE}.conf
# Start WireGuard
systemctl enable wg-quick@${WG_INTERFACE}
systemctl restart wg-quick@${WG_INTERFACE}
# Start Nebula
create_nebula_service
systemctl enable nebula
systemctl start nebula
# Setup firewall
setup_firewall
# Wait for WireGuard connection
echo -e "${YELLOW}[+] Waiting for WireGuard connection...${NC}"
sleep 3
# Wait for Nebula connection
echo -e "${YELLOW}[+] Waiting for Nebula connection...${NC}"
sleep 5
# Test connection to cluster
if ! ping -c 1 -W 2 10.10.0.1 > /dev/null 2>&1; then
echo -e "${RED}Failed to connect to cluster via WireGuard!${NC}"
echo "Please check the cluster details and firewall settings."
# Test connection to lighthouse
if ! ping -c 1 -W 3 192.168.100.1 > /dev/null 2>&1; then
echo -e "${RED}Failed to connect to cluster via Nebula!${NC}"
echo "Please check the cluster secret and firewall settings."
exit 1
fi
echo -e "${GREEN}[✓] WireGuard connection established${NC}"
echo -e "${GREEN}[✓] Nebula connection established${NC}"
# Add this node to the cluster node's WireGuard config
echo -e "${YELLOW}[+] Requesting cluster to add this node as peer...${NC}"
# SSH to cluster node and add peer (requires SSH key setup)
ssh_cmd="wg set ${WG_INTERFACE} peer ${public_key} preshared-key <(echo ${preshared_key}) allowed-ips ${WG_NETWORK} persistent-keepalive 25"
echo -e "${YELLOW}Run this command on the cluster node (10.10.0.1) to add this peer:${NC}"
echo -e "${GREEN}sudo wg set ${WG_INTERFACE} peer ${public_key} preshared-key <(echo ${preshared_key}) allowed-ips ${WG_NETWORK} persistent-keepalive 25${NC}"
echo -e "${GREEN}sudo bash -c 'echo \"[Peer]\" >> /etc/wireguard/${WG_INTERFACE}.conf'${NC}"
echo -e "${GREEN}sudo bash -c 'echo \"PublicKey = ${public_key}\" >> /etc/wireguard/${WG_INTERFACE}.conf'${NC}"
echo -e "${GREEN}sudo bash -c 'echo \"PresharedKey = ${preshared_key}\" >> /etc/wireguard/${WG_INTERFACE}.conf'${NC}"
echo -e "${GREEN}sudo bash -c 'echo \"AllowedIPs = ${WG_NETWORK}\" >> /etc/wireguard/${WG_INTERFACE}.conf'${NC}"
echo -e "${GREEN}sudo bash -c 'echo \"PersistentKeepalive = 25\" >> /etc/wireguard/${WG_INTERFACE}.conf'${NC}"
read -p "Press enter once you've added this peer to the cluster node..."
# Register with cluster
echo "${my_ip} ${hostname} $(date)" >> "${NEBULA_CONFIG}/cluster-registry.txt"
# Join GlusterFS cluster
echo -e "${YELLOW}[+] Joining GlusterFS cluster...${NC}"
# Probe the cluster
gluster peer probe 10.10.0.1
# Probe the lighthouse
gluster peer probe 192.168.100.1
# Wait for peer to be connected
sleep 3
@ -254,17 +368,17 @@ EOF
gluster volume add-brick ${GLUSTER_VOLUME} replica ${new_replica_count} $(hostname):${GLUSTER_BRICK_PATH}/brick1 force
# Mount the volume
mount -t glusterfs 10.10.0.1:/${GLUSTER_VOLUME} ${GLUSTER_MOUNT_PATH}
mount -t glusterfs 192.168.100.1:/${GLUSTER_VOLUME} ${GLUSTER_MOUNT_PATH}
# Add to fstab
grep -q "${GLUSTER_VOLUME}" /etc/fstab || echo "10.10.0.1:/${GLUSTER_VOLUME} ${GLUSTER_MOUNT_PATH} glusterfs defaults,_netdev 0 0" >> /etc/fstab
grep -q "${GLUSTER_VOLUME}" /etc/fstab || echo "192.168.100.1:/${GLUSTER_VOLUME} ${GLUSTER_MOUNT_PATH} glusterfs defaults,_netdev 0 0" >> /etc/fstab
echo -e "\n${GREEN}════════════════════════════════════════${NC}"
echo -e "${GREEN}Successfully joined cluster!${NC}"
echo -e "${GREEN}════════════════════════════════════════${NC}\n"
echo -e "${YELLOW}Node details:${NC}"
echo " - WireGuard IP: ${my_ip}"
echo " - Public Key: ${public_key}"
echo " - Nebula IP: ${my_ip}"
echo " - Hostname: ${hostname}"
echo " - GlusterFS mounted at: ${GLUSTER_MOUNT_PATH}"
}
@ -272,16 +386,28 @@ EOF
show_status() {
echo -e "\n${YELLOW}=== Cluster Status ===${NC}\n"
if [ -f /etc/wireguard/${WG_INTERFACE}.conf ]; then
echo -e "${GREEN}WireGuard Status:${NC}"
wg show ${WG_INTERFACE}
if [ -f "${NEBULA_CONFIG}/config.yaml" ]; then
echo -e "${GREEN}Nebula Status:${NC}"
systemctl is-active nebula && echo "Service: Active" || echo "Service: Inactive"
if ip addr show nebula1 >/dev/null 2>&1; then
echo "Interface: nebula1 $(ip addr show nebula1 | grep 'inet ' | awk '{print $2}')"
else
echo "Interface: Not found"
fi
echo ""
if [ -f "${NEBULA_CONFIG}/cluster-registry.txt" ]; then
echo -e "${GREEN}Cluster Nodes:${NC}"
cat "${NEBULA_CONFIG}/cluster-registry.txt"
echo ""
fi
else
echo -e "${RED}WireGuard not configured${NC}\n"
echo -e "${RED}Nebula not configured${NC}\n"
fi
echo -e "${GREEN}GlusterFS Status:${NC}"
gluster peer status
gluster peer status 2>/dev/null || echo "Not connected to cluster"
echo ""
gluster volume status ${GLUSTER_VOLUME} 2>/dev/null || echo "Volume ${GLUSTER_VOLUME} not found"
echo ""