kickstart over cloud-init

README.md

@@ -1,3 +1,5 @@
+# Nullpoint
+
 <div align="center">
   <img src='./icon.svg' width="150px">
   <h2>nullpoint</h2>
@@ -14,7 +16,7 @@ Secure Fedora Server setup with LUKS encryption, TPM, and BTRFS RAID1 with focus
 - TPM-based boot verification
 - BTRFS RAID1 storage with optimized subvolumes
 - Automated deployment to Hetzner
-- Cloud-init based configuration
+- Kickstart-based automated installation
 
 ## Security Model
 
@@ -105,3 +107,49 @@ hcloud ssh-key create --name "fedora-server-hetzner" --public-key "$(cat ~/.ssh/
    lsblk
    clevis-luks-list -d /dev/sda2
    ```
+
+## Installation Process
+
+The installation is fully automated using Fedora's kickstart system:
+
+1. **Partitioning**:
+   - Boot partitions (1GB each) on both drives
+   - Main partitions using remaining space
+   - All partitions use BTRFS
+
+2. **Storage Setup**:
+   - RAID1 for boot partitions
+   - LUKS2 encryption for data partitions
+   - BTRFS RAID1 for data with optimized subvolumes
+
+3. **Security Setup**:
+   - TPM binding during installation
+   - Tang server integration
+   - Secure boot configuration
+
+4. **Post-Installation**:
+   - Automatic service configuration
+   - TPM update script installation
+   - System optimization
+
+## Troubleshooting
+
+### Installation Issues
+- Check installation logs at `/root/postinstall.log`
+- Press Alt+F3 during installation to view real-time logs
+- Press Alt+F1 to return to main installation screen
+
+### Boot Issues
+1. If TPM unlock fails:
+   - Use the manual passphrase from `/root/luks-passphrase.txt`
+   - Run `/root/update-tpm-bindings.sh` if firmware was updated
+
+2. If Tang server is unreachable:
+   - Check network connectivity
+   - Verify Tang server is running
+   - Use manual passphrase as fallback
+
+### Storage Issues
+- Check RAID status: `cat /proc/mdstat`
+- Check BTRFS status: `btrfs filesystem show`
+- Verify LUKS status: `cryptsetup status`

build-config.yaml.example

@@ -34,23 +34,7 @@ system:
       # 14: UEFI Runtime Services Data
       # 15: UEFI Secure Boot State
 
-  # Cloud-init Configuration
+  # System Settings
-  cloud_init:
+  timezone: UTC
-    timezone: UTC
+  keyboard: us
-    users:
+  language: en_US.UTF-8
-      - name: admin
-        groups: wheel
-        sudo: ALL=(ALL) NOPASSWD:ALL
-        ssh_authorized_keys:
-          - "your-ssh-key-here"
-    packages:
-      - btrfs-progs
-      - clevis
-      - clevis-luks
-      - clevis-tang
-      - clevis-tpm2
-      - tpm2-tools
-      - tpm2-tss
-      - cryptsetup
-      - systemd
-      - curl 

build.py

@@ -46,7 +46,7 @@ def generate_cloud_init(config):
         'users': cloud_init['users'],
         'package_update': True,
         'package_upgrade': True,
-        'packages': cloud_init['packages'],
+        'packages': cloud_init['packages'] + ['mdadm'],  # Add mdadm package
         'write_files': [
             {
                 'path': '/etc/clevis/tang.conf',
@@ -124,6 +124,25 @@ echo "Please reboot to verify the changes."
                 'permissions': '0700'
             }
         ],
+        # Use cloud-init's native disk setup for partitioning only
+        'disk_setup': {
+            '/dev/sda': {
+                'table_type': 'gpt',
+                'layout': [
+                    [1024, 'boot'],  # 1GB for boot
+                    ['auto', 'data']  # Rest for LUKS1
+                ],
+                'overwrite': True
+            },
+            '/dev/sdb': {
+                'table_type': 'gpt',
+                'layout': [
+                    [1024, 'boot'],  # 1GB for boot mirror
+                    ['auto', 'data']  # Rest for LUKS2
+                ],
+                'overwrite': True
+            }
+        },
         'runcmd': [
             # Check if passphrase is set
             '[ -n "$LUKS_PASSPHRASE" ] || { echo "Error: LUKS passphrase not found"; exit 1; }',
@@ -131,15 +150,14 @@ echo "Please reboot to verify the changes."
             f'curl -s -f {system["luks"]["tang_url"]}/adv > /dev/null || {{ echo "Tang server not accessible"; exit 1; }}',
             # Verify TPM is available
             'tpm2_getcap properties-fixed > /dev/null || { echo "TPM not available"; exit 1; }',
-            # Create filesystem
+            # Create boot RAID1
-            'mkfs.btrfs -f -d raid1 -m raid1 /dev/sda2 /dev/sdb2',
+            'mdadm --create --verbose /dev/md0 --level=1 --raid-devices=2 /dev/sda1 /dev/sdb1 --metadata=1.2',
-            'mount /dev/sda2 /mnt',
+            # Create filesystem on RAID array
-            # Create subvolumes
+            'mkfs.ext4 -L boot /dev/md0',
-            'btrfs subvolume create /mnt/@boot',
+            # Mount boot RAID
-            'btrfs subvolume create /mnt/@home',
+            'mkdir -p /mnt/boot',
-            'btrfs subvolume create /mnt/@db',
+            'mount /dev/md0 /mnt/boot',
-            'chattr +C /mnt/@db',
+            # Setup LUKS on data partitions
-            # Setup LUKS with escaped passphrase
             'echo -n "${LUKS_PASSPHRASE}" | tr -d "\n" | cryptsetup luksFormat /dev/sda2 --type luks2 --key-file -',
             'echo -n "${LUKS_PASSPHRASE}" | tr -d "\n" | cryptsetup luksFormat /dev/sdb2 --type luks2 --key-file -',
             # Setup Clevis with error handling
@@ -147,11 +165,23 @@ echo "Please reboot to verify the changes."
             'clevis luks bind -d /dev/sda2 tang -c /etc/clevis/tang.conf || { echo "Tang bind failed"; exit 1; }',
             'clevis luks bind -d /dev/sdb2 tpm2 -c /etc/clevis/tpm2.conf || { echo "TPM bind failed"; exit 1; }',
             'clevis luks bind -d /dev/sdb2 tang -c /etc/clevis/tang.conf || { echo "Tang bind failed"; exit 1; }',
+            # Open LUKS volumes
+            'echo -n "${LUKS_PASSPHRASE}" | tr -d "\n" | cryptsetup luksOpen /dev/sda2 root_a --key-file -',
+            'echo -n "${LUKS_PASSPHRASE}" | tr -d "\n" | cryptsetup luksOpen /dev/sdb2 root_b --key-file -',
+            # Create BTRFS on decrypted devices
+            'mkfs.btrfs -f -d raid1 -m raid1 /dev/mapper/root_a /dev/mapper/root_b',
+            'mount /dev/mapper/root_a /mnt',
+            # Create subvolumes
+            'btrfs subvolume create /mnt/@home',
+            'btrfs subvolume create /mnt/@db',
+            'chattr +C /mnt/@db',
             # Setup fstab
-            'echo "/dev/mapper/root / btrfs compress=zstd 0 0" >> /etc/fstab',
+            'echo "/dev/md0 /boot ext4 defaults 0 0" >> /etc/fstab',
-            'echo "/dev/mapper/root /boot btrfs subvol=@boot,compress=zstd 0 0" >> /etc/fstab',
+            'echo "/dev/mapper/root_a / btrfs compress=zstd 0 0" >> /etc/fstab',
-            'echo "/dev/mapper/root /home btrfs subvol=@home,compress=zstd 0 0" >> /etc/fstab',
+            'echo "/dev/mapper/root_a /home btrfs subvol=@home,compress=zstd 0 0" >> /etc/fstab',
-            'echo "/dev/mapper/root /db btrfs subvol=@db,nodatacow,noatime,compress=zstd 0 0" >> /etc/fstab',
+            'echo "/dev/mapper/root_a /db btrfs subvol=@db,nodatacow,noatime,compress=zstd 0 0" >> /etc/fstab',
+            # Save RAID configuration
+            'mdadm --detail --scan > /etc/mdadm.conf',
             # Enable services
             'systemctl enable clevis-luks-askpass.service',
             'systemctl enable clevis-luks-askpass.path'
@@ -161,35 +191,268 @@ echo "Please reboot to verify the changes."
     return yaml.dump(cloud_config)
 
 def download_fedora_image(config):
-    """Download Fedora Server cloud image."""
+    """Download Fedora Server netinstall image."""
     image = config['image']
-    url = f"https://download.fedoraproject.org/pub/fedora/linux/releases/{image['version']}/Cloud/x86_64/images/Fedora-Cloud-Base-{image['version']}-1.6.x86_64.qcow2"
+    url = f"https://download.fedoraproject.org/pub/fedora/linux/releases/{image['version']}/Server/x86_64/iso/Fedora-Server-netinst-x86_64-{image['version']}-1.6.iso"
     
-    cmd = ['curl', '-L', '-o', 'fedora-server.qcow2', url]
+    cmd = ['curl', '-L', '-o', 'fedora-server.iso', url]
     subprocess.run(cmd, check=True)
 
+def generate_kickstart(config):
+    """Generate kickstart configuration for automated installation."""
+    system = config['system']
+    
+    # TPM update script content
+    tpm_update_script = '''#!/bin/bash
+# Script to update TPM bindings after firmware updates
+# Usage: sudo ./update-tpm-bindings.sh
+
+set -e
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo "Please run as root"
+    exit 1
+fi
+
+# Check if TPM is available
+if ! tpm2_getcap properties-fixed > /dev/null; then
+    echo "Error: TPM not available"
+    exit 1
+fi
+
+# Check if Tang server is accessible
+TANG_URL=$(grep URL /etc/clevis/tang.conf | cut -d= -f2)
+if ! curl -s -f "$TANG_URL/adv" > /dev/null; then
+    echo "Error: Tang server not accessible"
+    exit 1
+fi
+
+# Get current PCR values
+echo "Current PCR values:"
+tpm2_pcrread sha256:$(cat /etc/clevis/tpm2.conf | jq -r '.pcr_ids' | tr -d '[]' | tr ',' ' ')
+
+# Ask for confirmation
+read -p "Have you updated firmware? Continue with TPM binding update? [y/N] " -n 1 -r
+echo
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo "Aborted"
+    exit 1
+fi
+
+# Update TPM bindings
+echo "Updating TPM bindings..."
+for dev in /dev/sda2 /dev/sdb2; do
+    echo "Processing $dev..."
+    
+    # Unbind old TPM binding
+    clevis luks unbind -d "$dev" -s 1 || true
+    
+    # Create new TPM binding
+    clevis luks bind -d "$dev" tpm2 -c /etc/clevis/tpm2.conf || {
+        echo "Error: Failed to bind TPM to $dev"
+        exit 1
+    }
+    
+    # Verify Tang binding
+    if ! clevis luks list -d "$dev" | grep -q "tang"; then
+        echo "Error: Tang binding not found on $dev"
+        exit 1
+    fi
+done
+
+echo "TPM bindings updated successfully!"
+echo "Please reboot to verify the changes."
+'''
+    
+    kickstart = f"""# Kickstart configuration for Fedora Server
+# Generated for Nullpoint
+
+# System language
+lang en_US.UTF-8
+
+# Keyboard layouts
+keyboard --vckeymap=us --xlayouts='us'
+
+# Network information
+network --bootproto=dhcp --device=link --activate
+
+# Root password
+rootpw --lock
+
+# System timezone
+timezone {system['timezone']} --utc
+
+# Installation type
+text
+
+# Wipe all disk
+zerombr
+bootloader --location=mbr --boot-drive=sda
+clearpart --all --initlabel
+
+# Disk partitioning information
+part btrfs.boot --fstype=btrfs --size=5120 --ondisk=sda
+part btrfs.boot --fstype=btrfs --size=5120 --ondisk=sdb
+part btrfs.main --fstype=btrfs --encrypted --grow --fsoptions="compress=zstd:1,space_cache=v2" --ondisk=sda
+part btrfs.main --fstype=btrfs --encrypted --grow --fsoptions="compress=zstd:1,space_cache=v2" --ondisk=sdb
+
+# BTRFS subvolumes
+btrfs /boot --label=fedora-boot btrfs.boot
+btrfs none --label=fedora-btrfs btrfs.main
+btrfs /home --subvol --name=home fedora-btrfs
+btrfs /db --subvol --name=db fedora-btrfs
+
+# Package source
+url --mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=fedora-$releasever&arch=$basearch
+repo --name=fedora
+repo --name=updates
+
+# Make sure initial-setup runs
+firstboot --reconfig
+
+# Package selection
+%packages
+@^server-product
+@system-tools
+btrfs-progs
+clevis
+clevis-luks
+clevis-tang
+clevis-tpm2
+tpm2-tools
+tpm2-tss
+cryptsetup
+systemd
+mdadm
+curl
+%end
+
+# Pre-installation script
+%pre
+# Create TPM and Tang config files
+mkdir -p /etc/clevis
+cat > /etc/clevis/tang.conf << EOF
+URL={system['luks']['tang_url']}
+Thumbprint={system['luks']['tang_thumbprint']}
+EOF
+
+cat > /etc/clevis/tpm2.conf << EOF
+{{
+    "pcr_bank": "{system['luks']['tpm']['pcr_bank']}",
+    "pcr_ids": "{','.join(map(str, system['luks']['tpm']['pcr_ids']))}"
+}}
+EOF
+%end
+
+# Post-installation script
+%post
+# https://unix.stackexchange.com/a/351755 for handling TTY in anaconda
+printf "\n=== Nullpoint Installation Progress ===\r\n" > /dev/tty1
+printf "Press Alt+F3 to view detailed installation logs\r\n" > /dev/tty1
+printf "Press Alt+F1 to return to main installation screen\r\n" > /dev/tty1
+printf "Current step: Setting up storage and encryption...\r\n\n" > /dev/tty1
+{{
+    # Generate secure passphrase
+    printf "Generating secure passphrase...\r\n" > /dev/tty1
+    LUKS_PASSPHRASE=$(openssl rand -base64 32)
+    echo "$LUKS_PASSPHRASE" > /root/luks-passphrase.txt
+    chmod 600 /root/luks-passphrase.txt
+
+    # Create RAID1 for boot
+    printf "Creating RAID1 array for boot...\r\n" > /dev/tty1
+    mdadm --create --verbose /dev/md0 --level=1 --raid-devices=2 /dev/sda1 /dev/sdb1 --metadata=1.2
+    mkfs.btrfs -f -L boot /dev/md0
+
+    # Setup LUKS on data partitions
+    printf "Setting up LUKS encryption...\r\n" > /dev/tty1
+    echo -n "$LUKS_PASSPHRASE" | tr -d "\n" | cryptsetup luksFormat /dev/sda2 --type luks2 --key-file -
+    echo -n "$LUKS_PASSPHRASE" | tr -d "\n" | cryptsetup luksFormat /dev/sdb2 --type luks2 --key-file -
+
+    # Setup Clevis
+    printf "Configuring Clevis for TPM and Tang...\r\n" > /dev/tty1
+    clevis luks bind -d /dev/sda2 tpm2 -c /etc/clevis/tpm2.conf
+    clevis luks bind -d /dev/sda2 tang -c /etc/clevis/tang.conf
+    clevis luks bind -d /dev/sdb2 tpm2 -c /etc/clevis/tpm2.conf
+    clevis luks bind -d /dev/sdb2 tang -c /etc/clevis/tang.conf
+
+    # Open LUKS volumes
+    printf "Opening LUKS volumes...\r\n" > /dev/tty1
+    echo -n "$LUKS_PASSPHRASE" | tr -d "\n" | cryptsetup luksOpen /dev/sda2 root_a --key-file -
+    echo -n "$LUKS_PASSPHRASE" | tr -d "\n" | cryptsetup luksOpen /dev/sdb2 root_b --key-file -
+
+    # Create BTRFS
+    printf "Creating BTRFS filesystem...\r\n" > /dev/tty1
+    mkfs.btrfs -f -d raid1 -m raid1 /dev/mapper/root_a /dev/mapper/root_b
+
+    # Create subvolumes
+    printf "Creating BTRFS subvolumes...\r\n" > /dev/tty1
+    mount /dev/mapper/root_a /mnt
+    btrfs subvolume create /mnt/@home
+    btrfs subvolume create /mnt/@db
+    chattr +C /mnt/@db
+
+    # Setup fstab
+    printf "Configuring system mount points...\r\n" > /dev/tty1
+    cat > /etc/fstab << EOF
+/dev/md0 /boot btrfs defaults 0 0
+/dev/mapper/root_a / btrfs compress=zstd 0 0
+/dev/mapper/root_a /home btrfs subvol=@home,compress=zstd 0 0
+/dev/mapper/root_a /db btrfs subvol=@db,nodatacow,noatime,compress=zstd 0 0
+EOF
+
+    # Save RAID configuration
+    printf "Saving RAID configuration...\r\n" > /dev/tty1
+    mdadm --detail --scan > /etc/mdadm.conf
+
+    # Enable services
+    printf "Enabling system services...\r\n" > /dev/tty1
+    systemctl enable clevis-luks-askpass.service
+    systemctl enable clevis-luks-askpass.path
+
+    # Create TPM update script
+    printf "Creating TPM update script...\r\n" > /dev/tty1
+    cat > /root/update-tpm-bindings.sh << 'EOF'
+{tpm_update_script}
+EOF
+
+    chmod +x /root/update-tpm-bindings.sh
+    printf "\nInstallation complete! The system will reboot shortly.\r\n" > /dev/tty1
+    printf "IMPORTANT: LUKS passphrase has been saved to /root/luks-passphrase.txt\r\n" > /dev/tty1
+}} 2>&1 | tee -a /root/postinstall.log > /dev/tty3
+%end
+"""
+    return kickstart
+
 def customize_image(config):
     """Customize the Fedora Server image."""
-    # Generate cloud-init config
+    # Generate kickstart config
-    cloud_init = generate_cloud_init(config)
+    kickstart = generate_kickstart(config)
     
-    # Create cloud-init ISO
+    # Create kickstart file
-    with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.ks') as f:
-        f.write(cloud_init)
+        f.write(kickstart)
         f.flush()
-        subprocess.run(['cloud-localds', 'cloud-init.iso', f.name], check=True)
         
-    # Customize image
+        # Create custom ISO with kickstart
-    cmd = [
+        cmd = [
-        'virt-customize',
+            'mkisofs',
-        '-a', 'fedora-server.qcow2',
+            '-o', 'fedora-server-custom.iso',
-        '--selinux-relabel',
+            '-b', 'isolinux/isolinux.bin',
-        '--run-command', 'dnf clean all',
+            '-c', 'isolinux/boot.cat',
-        '--run-command', 'dnf -y update',
+            '-boot-info-table',
-        '--run-command', 'dnf -y install cloud-init',
+            '-no-emul-boot',
-        '--copy-in', 'cloud-init.iso:/var/lib/cloud/seed/nocloud/'
+            '-boot-load-size', '4',
-    ]
+            '-R',
-    subprocess.run(cmd, check=True)
+            '-J',
+            '-v',
+            '-T',
+            '-V', 'Fedora-S-custom',
+            '-A', 'Fedora-S-custom',
+            'fedora-server.iso',
+            f.name
+        ]
+        subprocess.run(cmd, check=True)
 
 def create_snapshot(config):
     """Create new snapshot from image."""
@@ -198,7 +461,7 @@ def create_snapshot(config):
         'hcloud-upload-image', 'upload',
         '--architecture', image['hetzner_arch'],
         '--compression', 'xz',
-        '--image-path', 'fedora-server.qcow2',
+        '--image-path', 'fedora-server-custom.iso',
         '--name', image['name'],
         '--labels', f'os=fedora-server,version={image["version"]}',
         '--description', f'Fedora Server {image["version"]} for Nullpoint'

deploy-config.yaml.example

@@ -2,7 +2,10 @@
 hetzner:
   datacenter: nbg1
   server_type: cx31
-  ssh_key_name: fedora-coreos-hetzner
+  ssh_key_name: fedora-server-hetzner
+
+# Admin Configuration
+admin_ssh_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI..."  # Your SSH public key here
 
 # Hostname Configuration
 hostname:

deploy.py

@@ -94,27 +94,20 @@ def create_server(config, hostname, image_id):
     """Create a new server."""
     hetzner = config['hetzner']
     
-    # Generate secure passphrase
-    passphrase = generate_secure_passphrase()
-    
     # Generate cloud-init config for this specific server
     cloud_init = {
         'hostname': hostname,
         'timezone': 'UTC',
-        'users': config.get('users', []),
+        'users': [
-        'package_update': True,
-        'package_upgrade': True,
-        'write_files': [
             {
-                'path': '/root/luks-passphrase.txt',
+                'name': 'admin',
-                'content': passphrase,
+                'groups': ['wheel'],
-                'permissions': '0600'
+                'sudo': 'ALL=(ALL) NOPASSWD:ALL',
+                'ssh_authorized_keys': [config['admin_ssh_key']]
             }
         ],
-        'bootcmd': [
+        'package_update': True,
-            # Set passphrase before any LUKS operations
+        'package_upgrade': True
-            f'export LUKS_PASSPHRASE="{passphrase}"'
-        ]
     }
     
     # Create temporary cloud-init config
@@ -133,11 +126,10 @@ def create_server(config, hostname, image_id):
         ]
         subprocess.run(cmd, check=True)
         
-        # Print passphrase for user to save
+        print(f"\nServer '{hostname}' created successfully!")
-        print(f"\nIMPORTANT: Save this LUKS passphrase for {hostname}:")
+        print("The LUKS passphrase has been saved to /root/luks-passphrase.txt on the server.")
-        print(f"{passphrase}\n")
+        print("Please save this passphrase securely - it will be needed if TPM+Tang unlock fails.")
-        print("This passphrase will be needed if TPM+Tang unlock fails.")
+        print("\nYou can connect using: ssh admin@<server-ip>")
-        print("It is also saved in /root/luks-passphrase.txt on the server.")
 
 def get_server_ip(hostname):
     """Get the server's IP address."""