# Check if ID is in use
if grep -q \""${1}"\" /etc/pve/.vmlist; then
        printf "ID %s already in use...\n" "$1"
        STATUS_LIST=$(pct list | grep -E "^${1}" | tr -s ' ')
        STATUS=$(echo "$STATUS_LIST" | cut -d' ' -f2)
        NAME=$(echo "$STATUS_LIST" | cut -d' ' -f3)

        while true; do
                printf "Container %s (%s) is currently %s.\n" "$NAME" "$1" "$STATUS"
                read -rp "Continue and destroy container? (y/n) " yn
                case $yn in
                        [Yy]* ) break;;
                        [Nn]* ) exit;;
                esac
        done
else
        printf "ID %s is not in use. Continue...\n" "$1"
        EXISTING=0
fi

root@pve:~# /opt/pve-new-ct.sh 150
ID 150 already in use...
Container CT150 (150) is currently stopped.
Continue and destroy container? (y/n) y

root@pve:~# /opt/pve-new-ct.sh 151
ID 151 is not in use. Continue...

root@pve:~# /opt/pve-new-ct.sh 102
ID 102 already in use...
Container ns1 (102) is currently running.
Continue and destroy container? (y/n) n

# If this had a previous container destroy it first
if [ -z "$EXISTING" ]; then
        printf "Destroying CT %s...\n" "$1"
        pct destroy "$1" --purge || { echo "Failed to purge container, exiting..."; exit 1; }
fi

# Create new container
pct create "${1}" \
        zfs-dir:vztmpl/debian-12-standard_12.2-1_amd64.tar.zst \
        --storage zfs-main \
        --hostname "Testing" \
        --description "Testing description" \
        --onboot 1 \
        --cores 1 \
        --memory 256 \
        --swap 256 \
        --features nesting=1 \
        --timezone host \
        --unprivileged 1 \
        --net0 name=eth0,bridge=vmbr0,firewall=1,ip=192.168.30.150/24,gw=192.168.30.1,ip6=fd00:30::150/64,gw6=fd00:30::1 \
        --start 1 \
        || { echo "Failed to create container, exiting..."; exit 1; }


pct status "$1" --verbose

This script will need cleaned up so you can dynamically import things like hostname, needed memory, IP addresses, etc but for now it should be good enough.

3. Moving Files into the Container

houck@pve:~$ tree /zfs/lxc/
/zfs/lxc/
├── shared
│   └── opt
│       └── setup.sh
└── testing
    ├── opt
    │   └── hello-world.sh
    └── root
        └── hello-world.txt

6 directories, 3 files

# Copy needed files into container filesystem
printf "Copying files into CT %s...\n" "$1"
pct push "$1" /zfs/lxc/shared/opt/setup.sh /opt/setup.sh
pct push "$1" /zfs/lxc/testing/opt/hello-world.sh /opt/hello-world.sh
pct push "$1" /zfs/lxc/testing/root/hello-world.txt /root/hello-world.txt

# Copy needed files into container filesystem
printf "Copying files into CT %s...\n" "$1"
pct push "$1" /zfs/lxc/shared/opt/setup.sh /opt/setup.sh
pct push "$1" /zfs/lxc/testing/opt/hello-world.sh /opt/hello-world.sh
pct push "$1" /zfs/lxc/testing/root/hello-world.txt /root/hello-world.txt

# Run setup commands
printf "Running setup in CT %s...\n" "$1"
pct exec "$1" -- bash -c 'chmod +x /opt/*.sh'
pct exec "$1" -- bash -c '/opt/setup.sh'
pct exec "$1" -- bash -c '/opt/hello-world.sh'
pct exec "$1" -- bash -c 'cat /root/hello-world.txt'

houck@pve:~$ sudo /opt/pve-new-ct.sh 150
ID 150 already in use...
Container Testing (150) is currently running.
Continue and destroy container? (y/n) y
Shutting down CT 150...
Destroying CT 150...



Could not execute systemctl:  at /usr/bin/deb-systemd-invoke line 145.
Setting up ssh (1:9.2p1-2+deb12u3) ...
Processing triggers for man-db (2.11.2-2) ...
Processing triggers for libc-bin (2.36-9+deb12u3) ...
hello world

There is a lot more functionality I need to add to my setup script to create a fully fledged out system, but I’ll worry about that in step 5 once we are ready to start looking at “prod” deployments.

4. Gracefully Deleting the Previous Container

While messing with this I noticed that I was missing some nightly backups for my test LXC, 150. In the picture below notice I only have a backup from the 14th, while this container was first created mid-day on the 12th:

root@pve:/home/houck# pct help destroy
USAGE: pct destroy  [OPTIONS]

  Destroy the container (also delete all uses files).

...

  --purge       (default=0)
             Remove container from all related configurations. For example,
             backup jobs, replication jobs or HA. Related ACLs and Firewall
             entries will *always* be removed.

That is, in fact, the whole point. I’ve modified the script to no longer use the purge flag and now it no longer yeets all of my mostly-valid backups when rebuilding a container.

While toying with this I was curious if taking a Snapshot right before deletion would make sense. Because snapshots capture RAM and state theoretically if the new container goes awry this would make it easier to rebuild from scratch.

However snapshots do not persist when a container is deleted and re-created. They aren’t separate entities like full backups, but rather are linked to the container. Instead, I do think it would be wise to do a manual full backup right before destroying the container.

// Options mostly copied from `/etc/pve/jobs.cfg`
printf "Creating backup of CT %s...\n" "$1"
vzdump "$1" \
        --compress zstd \
        --fleecing 0 \
        --mailnotification always \
        --mailto pve@aidanhouck.com \
        --mode snapshot \
        --node pve \
        --notes-template "Pre-rebuild: {{guestname}}" \
        --storage zfs-dir \
        || { echo "Failed to backup container, exiting..."; exit 1; }

5. Final Primary Script Cleanup

Now that the big steps are sensible I want to finish cleaning up this script. I left a few hard coded IPs, files, etc in there that need to come out before moving a prod container.

CONFIG_ABOUT="/zfs/lxc/${2}/about.sh"
CONFIG_FILES="/zfs/lxc/${2}/files.sh"

# If did not supply path to container info, die
[ ! -f "${CONFIG_ABOUT}" ] && printf "ERROR: /zfs/lxc/%s/about.sh cannot be read!\n" "$2" && exit 1
[ ! -f "${CONFIG_FILES}" ] && printf "ERROR: /zfs/lxc/%s/files.sh cannot be read!\n" "$2" && exit 1

# shellcheck source=/dev/null
source "$CONFIG_ABOUT" || { printf "Failed to source %s, exiting..." "$CONFIG_ABOUT"; exit 1; }

...

# Use files.sh to copy needed files into the container and execute them
# shellcheck source=/dev/null
source "$CONFIG_FILES" || { printf "Failed to source %s, exiting..." "$CONFIG_FILES"; exit 1; }

root@pve:~# cat /zfs/lxc/testing/about.sh
#!/bin/bash

CT_NAME="Testing"
CT_DESC="Testing description"
CT_CORES="1"
CT_MEMORY="256"
CT_SWAP="256"
CT_UNPRIV="1"

CT_BRIDGE="vmbr0"
CT_IP4="192.168.30.150/24"
CT_IP4_GW="192.168.30.1"
CT_IP6="fd00:30::150/64"
CT_IP6_GW="fd00:30::1"

root@pve:~# cat /zfs/lxc/testing/files.sh
#!/bin/bash

# Copy needed files into container filesystem
printf "Copying files into CT %s...\n" "$1"
pct push "$1" /zfs/lxc/shared/opt/setup.sh /opt/setup.sh
pct push "$1" /zfs/lxc/testing/opt/hello-world.sh /opt/hello-world.sh
pct push "$1" /zfs/lxc/testing/root/hello-world.txt /root/hello-world.txt

# Run setup commands
printf "Running setup in CT %s...\n" "$1"
pct exec "$1" -- bash -c 'chmod +x /opt/*.sh'
pct exec "$1" -- bash -c '/opt/setup.sh'
pct exec "$1" -- bash -c '/opt/hello-world.sh'
pct exec "$1" -- bash -c 'cat /root/hello-world.txt'

The shellchecks are needed for SC1090, but besides that no real issues. Now I just need to make new directories and copy template/script files over for each container.

6. Putting It All Together

Finally it’s time to finally move a single container over just to finish the proof of concept up. I keep pretty decent notes in my wiki when setting something up for the first time, but they’re mostly interactive configuration. Things like config files, databases, and changes I forgot about are all going to be tough to deal with.

I’m going to start with Tailscale.

cp -r /zfs/lxc/shared /zfs/lxc/tailscale
mv /zfs/lxc/tailscale/about-template.sh /zfs/lxc/tailscale/about.sh
mv /zfs/lxc/tailscale/files-template.sh /zfs/lxc/tailscale/files.sh
mv /zfs/lxc/tailscale/opt/setup-sh /zfs/lxc/tailscale/opt/tailscale-setup.sh

Then tweaking about.sh and files.sh with reasonable values. I mostly just mirrored what my existing Tailscale container was running with.

# Install curl and Tailscale
apt-get install -y curl
curl -fsSL https://tailscale.com/install.sh | sh

# Make necessary changes for acting as an exit node
echo 'net.ipv4.ip_forward = 1' | tee -a /etc/sysctl.d/99-tailscale.conf
echo 'net.ipv6.conf.all.forwarding = 1' | tee -a /etc/sysctl.d/99-tailscale.conf
sysctl -p /etc/sysctl.d/99-tailscale.conf

# Change default config to allow tailscail to run
# in an unprivileged container
#cp /etc/default/tailscaled /etc/default/tailscaled.bak
#sed -iE 's/FLAGS.*$/FLAGS="--tun=userspace-networking --socks5-server=localhost:1055 --outbound-http-proxy-listen=localhost:1055"/' /etc/default/tailscaled

# Verify the status is started
systemctl start tailscaled

# Bring up with authkey
tailscale up --authkey=`cat /opt/tailscale-authkey` --advertise-routes="192.168.0.0/16" --advertise-exit-node --advertise-tags=tag:container --statedir=/var/lib/tailscale

tailscale status

/opt/tailscale-authkey was generated manually from https://login.tailscale.com/admin/settings/keys and expires after 90 days. I might look into auto-refreshes via API? Or potentially disabling expiration.

The new containers were not coming up as exit nodes due to not having the routes approved in the dashboard first:

    // Define the tags which can be applied to devices and by which users.
    "tagOwners": {
        "tag:container": ["autogroup:admin"],
    },
    
    "autoApprovers": {
        "routes": {
            "192.168.0.0/16": ["tag:container"],
        },
        "exitNode": ["tag:container"],
    },

# Create world read-writable dir for Tailscale to use
mkdir -p /zfs/lxc/tailscale/persist
chmod 777 tailscale/persist/

# Add to `tailscale-setup.sh`:
 --statedir=/persist/tailscale --state/persist/tailscale/tailscaled.state

# Needed because the default systemd service includes a
# '--state=/...' parameter, $FLAGS (from /etc/default/tailscaled)
# are placed AFTER the default parameter. Evidently Tailscale
# prefers the first duplicate flag defined, not the most recent...

mkdir /etc/systemd/system/tailscaled.service.d
echo '[Service]' > /etc/systemd/system/tailscaled.service.d/override.conf
echo 'ExecStart=' >> /etc/systemd/system/tailscaled.service.d/override.conf
echo 'ExecStart=/usr/sbin/tailscaled --socket=/run/tailscale/tailscaled.sock --port=${PORT} $FLAGS' >> /etc/systemd/system/tailscaled.service.d/override.conf

systemctl daemon-reload
systemctl restart tailscaled.service

# Add to `tailscale-files.sh`:
echo 'mp0: /zfs/lxc/tailscale/persist,mp=/persist' | tee -a /etc/pve/lxc/${1}.conf

Now when I delete and recreate the container it proper re-registers as the same, existing machine in the Tailscale dashboard. Testing seems good so now the last step is migrating the Prod container.

pct shutdown 111
rm -r /zfs/lxc/tailscale/persist/tailscale/*
vim /zfs/lxc/tailscale/about.sh
    -> edit non-conflicting IP addresses to intended IP addresses
    -> edit hostname to intended hostname
/opt/pve-new-ct.sh 101 tailscale

While this was in the middle of building I went ahead and deleted the old machine from Tailscale’s dashboard. This is easier than trying to migrate state when there is really nothing important to migrate.

After a few minutes it finished installation and popped up in the dashboard:

7. Extra: Hacking the PVE GUI

I like using the Proxmox GUI (I know, I know), so I figured it might be fun to try and hack on it to insert my own custom button for this script.

Looking online, some people have done similar things but not quite adding custom functionality: https://lunar.computer/news/customize-proxmox-60/

Ext.define('PVE.lxc.Config', {
...
    var moreBtn = Ext.create('Proxmox.button.Button', {
        text: gettext('More'),
        menu: {
 items: [
        {
            text: gettext('Clone'),
            iconCls: 'fa fa-fw fa-clone',
            hidden: !caps.vms['VM.Clone'],
            handler: function() {
            PVE.window.Clone.wrap(nodename, vmid, template, 'lxc');
            },
        },
        {
...

This looks like where the “More” dropdown for LXC containers is defined, which seems like a perfect place to add my own button.

Scouring around my local install it seems like all of these JS files get compiled into /usr/share/pve-manager/js/pvemanagerlib.js. Editing that file around line 34445 (Your mileage may vary…) I found the exact same code as above.

Adding my own item element worked very easily:

                {
                    text: gettext('Rebuild Container'),
                    disabled: template,
                    xtype: 'pveMenuItem',
                    iconCls: 'fa fa-terminal',
                    hidden: false,
                    confirmMsg: 'Are you sure?',
                    handler: function() {
                        Proxmox.Utils.API2Request({
                            url: base_url + '/template',
                            waitMsgTarget: me,
                            method: 'POST',
                            failure: function(response, opts) {
                                Ext.Msg.alert('Error', response.htmlStatus);
                            },
                        });
                    },
                },

This adds a button, but you may notice the handler: looks a bit off. This is where I got stuck. Somehow, you want this button to register and run a script on the server (/opt/pve-new-ct.sh $CT_ID $CT_NAME). I’m guessing you’ll need to determine where exactly the API is defined and add a new API endpoint specifically for this action, possibly even adding it as a job that gets shown with a proper status and everything.

That is a lot more work than I’m willing to entertain right now, so this is where I’ll leave it. There very well may be an easier way to hack this together without all of that footwork, but I wasn’t able to figure one out quite yet.

Conclusion

1. Learn a bit more about managing Proxmox via pct
2. Script some basic lxc creation, backups, and deletion
3. Build out shared scripting for common setup actions I was copy-pasting (auto updates, SSH, and email alerting)
4. Put it all together and fully “define” one of my containers

Hopefully some of this will be helpful, or at the very least interesting. I don’t endorse blindly copying my hackjob scripts but if you’re looking to do something similar and want some inspiration feel free.