F5 BIG-IP Backup Script

Here is my F5 tmos backup script. Going to walk through how to reproduce if I forget years later how to reproduce.

NOTE: My user is confback in this example

Create /home/confback/backups/bigip_backup.sh

Bash

#!/bin/sh
## PRE_REQ
## ssh-copy-id -i ~/.ssh/id_rsa.pub root@bigip
## TEST-VERIFY: ssh root@bigip <--no password login means success
###### SYNTAX to RUN: ./bigip_backup.sh bigip_devices.txt [daily|weekly]
## BACKUP FILES ARE DELIMITED BY - [hostname-f5_daily_backup-DATE-TIME]
#DEBUG=yes
tstart=$SECONDS && [[ -n $DEBUG ]] && echo "DEBUG: TOTAL start is $tstart"
##CLEANUP JUMPBOX BEFORE ADDING MORE
#

cat $1 | while read REMOTE_BIGIP || [[ -n $REMOTE_BIGIP ]];
do
    echo "********************** $REMOTE_BIGIP STARTING ******************************"
    start=$SECONDS && [[ -n $DEBUG ]] && echo "DEBUG: start is $start"
    DATETIME="`date +%Y%m%d-%H%M`"
    REMOTE_PATH="/var/tmp"
    LOCAL_PATH="/home/confback/backups"
    FILE_UCS="$(echo $REMOTE_BIGIP | cut -d'.' -f1)-f5_daily_backup-${DATETIME}.ucs"
    FILE_SCF="$(echo $REMOTE_BIGIP | cut -d'.' -f1)-f5_daily_backup-${DATETIME}.scf"
    FILE_CERT="$(echo $REMOTE_BIGIP | cut -d'.' -f1)-f5_daily_backup-${DATETIME}.cert.tar"
    start=$SECONDS

    if [ $# -eq 0 ];
    then
    echo "$0: Missing BIGIP FQDN - Try Running again: .bigip_backup.sh txsat1slbdv03.ally.corp"
    exit 1
    elif [ $# -gt 2 ];
    then
        echo "$0: Too many arguments: $@"
        exit 1
    else
    echo "=================================================================="
    echo "filename........: $1"
    echo "REMOTE_BIGIP....: $REMOTE_BIGIP"
    echo "DATETIME........: $DATETIME"
    echo "REMOTE_PATH.....: $REMOTE_PATH"
    echo "LOCAL_PATH......: $LOCAL_PATH"
    echo "FILE_UCS........: $FILE_UCS"
    echo "FILE_SCF........: $FILE_SCF"
    echo "FILE_CERT.......: $FILE_CERT"
    echo "=================================================================="
    echo "Variable are SET"
    echo ""
    fi
    echo "BIG-IP: $REMOTE_BIGIP"
    echo "-----------------------------------------------------"
    echo ""
#    cmd=ssh -n root@$REMOTE_BIGIP 'bash -s BEFORE' < ltmchgverify.sh
#    $cmd || { echo '$cmd' ; exit 1; } 
    ####################
    ## TOTALVIPS Section ##
#    echo "-----------------------------------------------------"
#    echo "BEGIN SECTION TOTALVIPS"
#    echo "-----------------------------------------------------"
#    echo "`date +%Y%m%d_%H.%M.%S`: ENTERING TOTALVIPS SECTION for $REMOTE_BIGIP"
#    echo ""
#    totalvips=$(ssh -n root@$REMOTE_BIGIP tmsh -q -c 'cd / ; list ltm recursive' | grep -c "ltm virtual")
#    echo "Total VIPs for $REMOTE_BIGIP is $totalvips"
#    echo "`date +%Y%m%d_%H.%M.%S`: DONE WITH TOTALVIPS Section for $REMOTE_BIGIP"
#    echo "-----------------------------------------------------"
#    echo "END SECTION TOTALVIPS"
#    echo "-----------------------------------------------------"
#    echo ""
    #############################
    ## CONNECTION TEST SECTION ##
    echo "-----------------------------------------------------"
    echo "BEGIN SECTION CONNECTION TEST"
    echo "-----------------------------------------------------"
    echo "ENTERING CONNECTION TEST for $REMOTE_BIGIP"
    echo ""
    echo "Can we connect to $REMOTE_BIGIP without a password prompt?"
    cmd="ssh -n root@$REMOTE_BIGIP > /dev/null"
    echo "`date +%Y%m%d_%H.%M.%S`: running $cmd"
    #$cmd || { echo '$cmd' ; exit 1; }
    expect -c "
        set timeout 5
        spawn $cmd
        expect "Password" { exit 1 }
        "
    # Do something based on the exit status of the expect script.
    if [ $? -eq 1 ]; then
        echo "$?"
        echo "Password prompt detected on $REMOTE_BIGIP "
        echo "run ssh-copy-id -i ~/.ssh/id_rsa.pub root@$REMOTE_BIGIP "
        echo "then run the script again and you should be good"
        echo "exiting script for now for $REMOTE_BIGIP but going to next device in list"
        exit
    fi
    echo "DONE with CONNECTION TEST FOR $REMOTE_BIGIP "
    echo "-----------------------------------------------------"
    echo "END SECTION CONNECTION TEST"
    echo "-----------------------------------------------------"
    echo ""
    ####################
    ## UPTIME Section ##
    echo "-----------------------------------------------------"
    echo "BEGIN SECTION UPTIME"
    echo "-----------------------------------------------------"
    echo "`date +%Y%m%d_%H.%M.%S`: ENTERING UPTIME SECTION for $REMOTE_BIGIP"
    echo ""
    uplength=$(ssh -n root@$REMOTE_BIGIP uptime)
    echo "Uptime for $REMOTE_BIGIP is $uplength"
    echo "`date +%Y%m%d_%H.%M.%S`: DONE WITH UPTIME Section for $REMOTE_BIGIP"
    echo "-----------------------------------------------------"
    echo "END SECTION UPTIME"
    echo "-----------------------------------------------------"
    echo ""
    ######################
    ## MASTERKEY Section ##
    echo "-----------------------------------------------------"
    echo "BEGIN SECTION MASTERKEY"
    echo "-----------------------------------------------------"
    echo "`date +%Y%m%d_%H.%M.%S`: ENTERING MASTERKEY SECTION for $REMOTE_BIGIP"
    echo ""
    masterkey=$(ssh -n root@$REMOTE_BIGIP f5mku -K)
    echo "MasterKey for $REMOTE_BIGIP is $masterkey"
    echo "`date +%Y%m%d_%H.%M.%S`: DONE WITH MASTERKEY Section for $REMOTE_BIGIP"
    echo "-----------------------------------------------------"
    echo "END SECTION MASTERKEY"
    echo "-----------------------------------------------------"
    echo ""
    ######################
    ## SOFTWARE Section ##
    echo "-----------------------------------------------------"
    echo "BEGIN SECTION SOFTWARE"
    echo "-----------------------------------------------------"
    echo "ENTERING SOFTWARE SECTION for $REMOTE_BIGIP"
    echo ""
    software=$(ssh -n root@$REMOTE_BIGIP tmsh show sys software status | grep "yes  complete")
    #echo "`date +%Y%m%d_%H.%M.%S`: running $cmd"
    #$cmd || { echo '$cmd' ; exit 1; }
    echo "SOFTWARE installed: $software"
    echo "DONE WITH SOFTWARE Section for $REMOTE_BIGIP"
    echo "-----------------------------------------------------"
    echo "END SECTION SOFTWARE"
    echo "-----------------------------------------------------"
    echo ""
    #################
    ## HDD Section ##
    ## CLEANUP DRIVE TO MAKE ROOM FOR UCS FILE
    echo "-----------------------------------------------------"
    echo "BEGIN SECTION HDD"
    echo "-----------------------------------------------------"
    echo "ENTERING CLEANING UP DRIVE for $REMOTE_BIGIP"
    echo ""
    echo "How full is /var/tmp?"
    ssh -n root@$REMOTE_BIGIP df -P /var/tmp | tail -1 | awk '{ print $5}'
    echo "`date +%Y%m%d_%H.%M.%S`: time to cleanup created files and rpm-tmp files"
    cmd="ssh -n root@$REMOTE_BIGIP rm -f $REMOTE_PATH/{rpm-tmp.*,*.ucs,*qkview,*.*cap} > /dev/null"
    $cmd || { echo '$cmd' ; exit 1; }
    #echo "After Cleanup, Do we have more space for a backup?"
    hdd=$(ssh -n root@$REMOTE_BIGIP df -P /var/tmp | tail -1 | awk '{ print $5}')
    echo ""
    echo "DONE WITH CLEANING UP DRIVE for $REMOTE_BIGIP"
    echo "-----------------------------------------------------"
    echo "END SECTION HDD"
    echo "-----------------------------------------------------"
    echo ""
    ##########################
    ## SAVE AND TEST CONFIG ##
    echo "-----------------------------------------------------"
    echo "BEGIN SECTION CONFIG"
    echo "-----------------------------------------------------"
    echo "ENTERING SAVE AND VERIFY CONFIG SECTION for $REMOTE_BIGIP"
    echo ""
    echo "Saving Config on $REMOTE_BIGIP first"
    cmd="ssh -n root@$REMOTE_BIGIP tmsh save /sys config > /dev/null"
    echo "`date +%Y%m%d_%H.%M.%S`: running $cmd"
    echo "done saving config" 
    echo ""
    echo "Let's VERIFY the config"
    cmd="ssh -n root@$REMOTE_BIGIP tmsh load sys config verify partitions all"
    echo "`date +%Y%m%d_%H.%M.%S`: running $cmd"
    echo "Done with the VERIFY config"
    echo ::
    echo "DONE WITH SAVE AND VERIFY CONFIG for $REMOTE_BIGIP"
    echo "-----------------------------------------------------"
    echo "END SECTION CONFIG"
    echo "-----------------------------------------------------"
    echo ""
    ##############################
    ## NOW let's create the UCS ##
    echo "-----------------------------------------------------"
    echo "BEGIN SECTION CREATE UCS"
    echo "-----------------------------------------------------"
    echo "ENTERING CREATING UCS BACKUP FILE for $REMOTE_BIGIP"
    echo ""
    echo "Creating Backup File (.ucs) for $REMOTE_BIGIP"
    cmd="ssh -n root@$REMOTE_BIGIP tmsh save /sys ucs $REMOTE_PATH/$FILE_UCS > /dev/null"
    echo "`date +%Y%m%d_%H.%M.%S`: running $cmd"
    $cmd || { echo '$cmd' ; exit 1; }
    if [ $? -eq 0 ]; then
        echo "success running $cmd"
    else
        echo "failed running $cmd"
        exit
    fi
    echo "DONE WITH CREATING UCS BACKUP FILE for $REMOTE_BIGIP "
    echo "-----------------------------------------------------"
    echo "END SECTION CREATE UCS"
    echo "-----------------------------------------------------"
    echo ""
    #################################
    ## NOW let's create the QKVIEW ##
    echo "-----------------------------------------------------"
    echo "BEGIN SECTION CREATE QKVIEW"
    echo "-----------------------------------------------------"
    echo "ENTERING CREATING QKVIEW DIAG FILE for $REMOTE_BIGIP"
    echo ""
    echo "Creating Diagnostic File (.qkview) for $REMOTE_BIGIP"
    cmd="ssh -n root@$REMOTE_BIGIP qkview -s0 -o asm-request-log > /dev/null"
    echo "`date +%Y%m%d_%H.%M.%S`: running $cmd"
    $cmd || { echo '$cmd' ; exit 1; }
    if [ $? -eq 0 ]; then
        echo "success running $cmd"
    else
        echo "failed running $cmd"
        exit
    fi
    echo "DONE WITH CREATING QKVIEW DIAG FILE for $REMOTE_BIGIP "
    echo "-----------------------------------------------------"
    echo "END SECTION CREATE QKVIEW"
    echo "-----------------------------------------------------"
    echo ""
    ######################
    ## DOWNLOAD Section ##
    echo "-----------------------------------------------------"
    echo "BEGIN SECTION DOWNLOAD"
    echo "-----------------------------------------------------"
    echo "ENTERING DOWNLOAD SECTION"
    echo ""
    echo "Downloading .ucs from $REMOTE_BIGIP"
    cmd="scp -v root@$REMOTE_BIGIP:$REMOTE_PATH/*f5_daily_backup*.ucs $LOCAL_PATH/f5"
    echo "`date +%Y%m%d_%H.%M.%S`: running $cmd"
    $cmd > /dev/null || { echo '$cmd' ; exit 1; }
    echo "Finished Downloading ucs"
    echo ""
    echo "Let's remove the .ucs file from $REMOTE_BIGIP to save space on the drive"
    cmd="ssh -n root@$REMOTE_BIGIP rm -f $REMOTE_PATH/*.ucs > /dev/null"
    echo "`date +%Y%m%d_%H.%M.%S`: running $cmd"
    $cmd || { echo '$cmd' ; exit 1; }
    echo ""
    echo "Downloading .qkview from $REMOTE_BIGIP"
    cmd="scp -v root@$REMOTE_BIGIP:$REMOTE_PATH/*.qkview $LOCAL_PATH/f5/diag"
    echo "`date +%Y%m%d_%H.%M.%S`: running $cmd"
    $cmd > /dev/null || { echo '$cmd' ; exit 1; }
    echo "Finished Downloading qkview"
    echo ""
    echo "Let's remove the .qkview file from $REMOTE_BIGIP to save space on the drive"
    cmd="ssh -n root@$REMOTE_BIGIP rm -f $REMOTE_PATH/*.qkview > /dev/null"
    echo "`date +%Y%m%d_%H.%M.%S`: running $cmd"
    $cmd || { echo '$cmd' ; exit 1; }
    echo "DONE WITH DOWNLOAD SECTION for $REMOTE_BIGIP "
    echo "-----------------------------------------------------"
    echo "END SECTION DOWNLOAD"
    echo "-----------------------------------------------------"
    echo ""
    ####################
    ## TOTALVIPS Section ##
#    echo "-----------------------------------------------------"
#    echo "BEGIN SECTION TOTALVIPS"
#    echo "-----------------------------------------------------"
#    echo "`date +%Y%m%d_%H.%M.%S`: ENTERING TOTALVIPS SECTION for $REMOTE_BIGIP"
#    echo ""
#    totalvips=$(ssh -n root@$REMOTE_BIGIP (tmsh -q -c 'cd / ; show ltm virtual recursive' | egrep 'Ltm::Virtual|Availability' | grep -c "Ltm::Virtual"))
#    echo "Total VIPs for $REMOTE_BIGIP is $totalvips"
#    echo "`date +%Y%m%d_%H.%M.%S`: DONE WITH TOTALVIPS Section for $REMOTE_BIGIP"
#    echo "-----------------------------------------------------"
#    echo "END SECTION TOTALVIPS"
#    echo "-----------------------------------------------------"
#    echo ""
    #GENERAL
    #echo "get $REMOTE_BIGIP:$REMOTE_PATH" | sftp root@$REMOTE_BIGIP:$REMOTE_PATH/*f5_daily_backup*.* $LOCAL_PATH <<EOF
    #echo "`date +%Y%m%d_%H.%M.%S`: time to cleanup created files and rpm-tmp files"
    #$cmd="ssh -n root@$REMOTE_BIGIP rm -f $REMOTE_PATH/{rpm-tmp.*,*f5_daily_backup*.*} > /dev/null"
    #$cmd || { echo '$cmd' ; exit 1; }
    echo ""
    echo "FINISHED with $REMOTE_BIGIP"
    echo "*********************************************************************************************"
    echo "OVERVIEW for BIG-IP: $REMOTE_BIGIP"
    echo "MASTERKEY: $masterkey"
    echo "UPTIME: $uplength"
    echo "SOFTWARE: $software"
    echo "HDD FULL%: $hdd"
#    echo "Number of Virtual Servers: $totalvips"
    echo ""
    echo "THIS BACKUP TOOK...."
    duration=$(( SECONDS - start ))
    echo "Duration(seconds): $duration"
    echo "Duration(minutes): $(( $duration / 60))"
    echo "********************** $REMOTE_BIGIP FINISHED ******************************"
    echo "*********************************************************************************************"
done
#####################
## CLEANUP JUMPBOX ##
echo "You Should ONLY see this after all big-ips are done backing up"
echo "Cleaning up any backup files older than 30 days on RHEL storage"
echo "...................................."
echo "Removing UCS files older than 30days..."
find /home/confback/backups -name "*.ucs" -type f -mtime +31 -delete
echo "Done with removing old UCS files"
echo ""
echo "Removing LOG files older than 30days..."
find /home/confback/backups -name "*.log" -type f -mtime +31 -delete
echo "Done with removing old LOG files"
echo ""
echo "Removing IB files older than 30days..."
find /home/confback/backups -name "*.gz" -type f -mtime +31 -delete
echo "Done with removing old IB files"
echo ""
echo "FINISHED CLEANING UP JUMPBOX and now exiting script"
echo "...................................."
echo ""
################################
## VERIFY WE HAVE ALL BACKUPS ##
echo "ENTERING THE VERIFY WE HAVE ALL BACKUPS"
echo ""
location=usuyk1
echo "$location should have 4 devices"
result=$(find /home/confback/backups/f5 -maxdepth 2 -name "$location*-f5_daily_backup-`date +%Y%m%d`*" | wc -l)
echo "Number of usuyk1 devices backed up should be 4 and is $result"
if [ "$result" = "4" ]; then
    echo "We have all $result $location devices backed up"
else
    echo "Only found $result $location devices backed up when it should be 4"
fi
echo ""
location=txdfw1
echo "$location should have 16 devices"
result=$(find /home/confback/backups/f5 -maxdepth 2 -name "$location*-f5_daily_backup-`date +%Y%m%d`*" | wc -l)
echo "Number of $location devices backed up should be 16 and is $result"
if [ "$result" = "16" ]; then
    echo "We have all $result $location devices backed up"
else
    echo "Only found $result $location devices backed up when it should be 16"
fi
echo ""
location=txsat1
echo "$location should have 40 devices"
result=$(find /home/confback/backups/f5 -maxdepth 2 -name "$location*-f5_daily_backup-`date +%Y%m%d`*" | wc -l)
echo "Number of $location devices backed up should be 40 and is $result"
if [ "$result" = "40" ]; then
    echo "We have all $result $location devices backed up"
else
    echo "Only found $result $location devices backed up when it should be 40"
fi
    echo "/////////////// DURATION of SCRIPT SUMMARY ///////////////"
    echo ""
    echo "Script took this long to run on all the devices included in file: $1"
    tduration=$(( SECONDS - tstart ))
    echo "TOTAL Duration(seconds): $tduration"
    echo "TOTAL Duration(minutes): $(( $tduration / 60))"
    echo ""
    echo "NOW KICKING OFF SCRIPT TO UPLOAD QKVIEWS TO iHEALTH"
    /usr/bin/bash /home/confback/backups/f5/diag/upqkview.sh
    echo "BYE"

Now you need to create a text file with your all your devices in it on all seperate lines

CREATE /home/confback/backups/bigip_devices_prd.txt

NOTE: My user is confback in this example

`usfnt1slbprd01.ally.corp
usfnt1slbprd02.ally.corp`

**NOTE: you can have whatever FQDN’s in the .txt files you want.. no limit from my testing so far. Just make sure you have one per line. NEXT create the cronjob usually by running crontab -e and adding these three lines

Bash

0 0 * * * /home/confback/backups/bigip_backup.sh /home/confback/backups/bigip_devices_dev.txt > /home/confback/backups/bigip_backup_dev_`date +20\%y\%m\%d_\%H\%M\%S`.log

Now every day you’ll get backup of the devices you included in your text file and you will get a log that is layed out to make it easier to do searches for info you might want.

Example: You want all the devices hostname and version of software just run this command

Bash

egrep -Rw *.log -e 'BIG-IP:|yes complete'

Another example is to give you the full output of a SECTION and you can use awk for this and you can use this example to print everything from BEGIN of section of text file to END of section of text file.

This example we will search for SOFTWARE SECTION as an example

Bash

awk '/^BEGIN SECTION SOFTWARE/,/^END SECTION SOFTWARE/ {if (!/^BEGIN SECTION SOFTWARE/ && !/^END SECTION SOFTWARE/) print}' *.log

These are just some examples of how to backup your configs from your BIG-IP boxes to a jumpbox in this case running RHEL.

Here are the instructions to create the file that will upload the qkview diagnostic files to iHealth using the new API method.

NOTE: My user is still confback in these examples

CREATE file named /home/confback/backups/f5/diag/upqkview.sh and paste the following:

Bash

#!/bin/sh
## PRE_REQ
#!/bin/sh
## PRE_REQ

ls -f /home/confback/backups/f5/diag/*.qkview | while read FILE_QKVIEW || [[ -n $FILE_QKVIEW ]]; do
    if [ -z $FILE_QKVIEW ]; then
        echo "no value in FILE_QKVIEW"
        break
    fi

    start=$SECONDS
    echo "***********************************************"
    echo "STARTING with $FILE_QKVIEW"
    DATETIME="`date +%Y%m%d_%H%M`"
    REMOTE_PATH='/var/tmp'
    LOCAL_PATH="/home/confback/backups/f5/diag"
    BEARERTOKEN=`(curl --request POST --url https://identity.account.f5.com/oauth2/ausp95ykc80HOU7SQ357/v1/token -H "accept:application/json" -H "authorization: Basic <paste your authorization token here you got from your iHealth profile>" -H "cache-control: no-cache" -H "content-type: application/x-www-form-urlencoded" --data "grant_type=client_credentials&scope=ihealth" | jq -r .access_token)`
    echo "$FILE_QKVIEW"
    echo "***********************************************"
    echo "NOW let's upload $FILE_QKVIEW to iHealth"
    curl -v -H "Authorization: Bearer "$BEARERTOKEN -H "Accept: application/vnd.f5.ihealth.api" --user-agent "MyGreatiHealthClient" -o - -F qkview=@"$FILE_QKVIEW" 'https://ihealth2-api.f5.com/qkview-analyzer/api/qkviews?visible_in_gui=true'
    echo "done uploading $FILE_QKVIEW to iHealth"
    echo ""
    echo "***********************************************"
    echo "DELETE local qkview that was uploaded"
#    rm -rf $FILE_QKVIEW
    echo "done removing $FILE_QKVIEW"
    echo ""
    echo "***********************************************"
    echo "done with $FILE_QKVIEW..  continue to next F5"
    echo "***********************************************"
done
rm -rf /home/confback/backups/f5/diag/*.qkview
echo "done with uploading qkviews to ihealth and cleaning up local drive"

NOTE: you will need to replace the authorization in the BEARERTOKEN variable. You get this from iHealth under your profile settings.

Related Articles

Responses