Pages

AIX OS Upgrades with NIM and nimadm

Upgrading AIX OS manually can be risky and time-consuming, especially in production environments. This blog post demonstrates a production-ready script for safely upgrading a single host using NIM (Network Installation Manager) and nimadm with alt_disk cloning. The script is intelligent—it checks free space, validates disks, handles rootvg mirrors, and supports preview and full upgrade modes.

Complete Script
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Preview Mode
#!/usr/bin/ksh
#===============================================================================
#
# aix_os_upgrade-one-host.ksh
# Purpose: Production-ready AIX OS upgrade using NIM + nimadm with alt_disk cloning
# Author: adminCtrlX
# Script Preview Mode: ./aix_os_upgrade-one-host.ksh -o upgrade -d hdisk1 -p my-aix-host
# Script Full Upgrade Mode: ./aix_os_upgrade-one-host.ksh -o upgrade -d hdisk1 -f my-aix-host
#=========================================================================
set -o errexit
set -o nounset
set -o pipefail 2>/dev/null || true

SCRIPT_NAME=$(basename "$0")
LOG_DIR="/var/log/aix_upgrade"
mkdir -p "$LOG_DIR"

#-------------------------
# Parameters / Defaults
#-------------------------
HOST=""
OPERATIONS=""
TARGET_DISK=""
ALT_DISK_FLAGS=""
TARGET_OS="7300"
SPOT_NAME="spot_7300_01_00"
LPP_NAME="lpp_7300_01_00"
NIMADM_VG="cachevg"
PREVIEW=1
VERIFY=0
FORCE=0
LOG_FILE=""

EMAIL_RECIPIENTS="sysadm@ppc.com"
EMAIL_SUBJECT_SUCCESS="AIX OS Upgrade SUCCESS: $HOST"
EMAIL_SUBJECT_FAILURE="AIX OS Upgrade FAILURE: $HOST"

#-------------------------
# Logging
#-------------------------
log() { print -- "$(date '+%F %T') : $*" | tee -a "$LOG_FILE"; }

fatal() {
log "FATAL: $*"
[[ -n "$LOG_FILE" && -f "$LOG_FILE" ]] && email_notify "FAILURE" "$EMAIL_SUBJECT_FAILURE"
exit 1
}

email_notify() {
local status="$1"
local subject="$2"
if command -v mail >/dev/null 2>&1; then
cat "$LOG_FILE" | mail -s "$subject" "$EMAIL_RECIPIENTS"
log "Email notification sent: $status"
else
log "Mail command not found — cannot send $status email"
fi
}

#-------------------------
# Usage
#-------------------------
usage() {
print "
Usage:
$SCRIPT_NAME -o upgrade -d <hdisk> -t <target_os> -S <spot_name> -L <lpp_name> [options] <hostname>

Options:
-o Operation (upgrade)
-d Target disk (hdisk1)
-t Target OS level (7300)
-S NIM spot name
-L LPP name
-A alt_disk flags (e.g., -g)
-p Preview mode (default)
-v Verify only
-f Force execution (disable preview)
"
exit 1
}

#-------------------------
# Argument parsing
#-------------------------
while getopts ":o:d:t:S:L:A:pvf" opt; do
case "$opt" in
o) OPERATIONS="$OPTARG" ;;
d) TARGET_DISK="$OPTARG" ;;
t) TARGET_OS="$OPTARG" ;;
S) SPOT_NAME="$OPTARG" ;;
L) LPP_NAME="$OPTARG" ;;
A) ALT_DISK_FLAGS="$OPTARG" ;;
p) PREVIEW=1 ;;
v) VERIFY=1 ;;
f) FORCE=1 ; PREVIEW=0 ;;
*) usage ;;
esac
done
shift $((OPTIND - 1))
HOST="${1:-}"

[[ -n "$HOST" && -n "$OPERATIONS" && -n "$TARGET_DISK" ]] || usage
[[ "$OPERATIONS" = "upgrade" ]] || fatal "Only 'upgrade' operation is supported"

LOG_FILE="$LOG_DIR/${HOST}.log"
[[ $(id -u) -eq 0 ]] || fatal "Must be run as root"

#-------------------------
# Connectivity check
#-------------------------
check_connectivity() {
log "Checking connectivity to $HOST"
ping -c 1 "$HOST" >/dev/null 2>&1 || fatal "Ping failed"
ssh "$HOST" true >/dev/null 2>&1 || fatal "SSH failed"
log "Connectivity OK"
}

#-------------------------
# NIM client check
#-------------------------
check_nim_client() {
log "Checking if $HOST is a defined NIM client"
lsnim -l "$HOST" >/dev/null 2>&1 || fatal "$HOST is not a NIM client"
log "$HOST is a valid NIM client"
}

#-------------------------
# Check cachevg free space vs client rootvg
#-------------------------
check_cachevg_space() {
log "Checking client rootvg size and NIM server $NIMADM_VG free space"

ROOTVG_MB=$(ssh "$HOST" lsvg rootvg | awk '
NR==2 {pp=$6}
NR>2 && $1~/^[0-9]+$/ {t+=$3}
END {print t*pp}')
[[ -n "$ROOTVG_MB" && "$ROOTVG_MB" -gt 0 ]] || fatal "Cannot determine client rootvg size"
log "Client rootvg size: $ROOTVG_MB MB"

CACHEVG_FREE_MB=$(lsvg -l "$NIMADM_VG" | awk '
NR==2 {pp=$6}
NR>2 && $1~/^[0-9]+$/ {f+=$6}
END {print f*pp}')
[[ -n "$CACHEVG_FREE_MB" ]] || fatal "Cannot determine NIM cachevg free space"
log "NIM server cachevg free: $CACHEVG_FREE_MB MB"

[[ "$CACHEVG_FREE_MB" -ge "$ROOTVG_MB" ]] || fatal "Insufficient cachevg free space"
}

#-------------------------
# Pre-flight checks
#-------------------------
preflight_checks() {
log "Running pre-flight checks on $HOST"
ssh "$HOST" bash -s >>"$LOG_FILE" 2>&1 <<EOF
for cmd in nimadm alt_disk_install oslevel lspv lsvg bootlist chdev unmirrorvg reducevg chpv ipl_varyon bosboot; do
command -v \$cmd >/dev/null 2>&1 || { echo "Command \$cmd missing"; exit 1; }
done
lspv | awk '{print \$1}' | grep -w "$TARGET_DISK" >/dev/null 2>&1 || { echo "Disk $TARGET_DISK not found"; exit 1; }
EOF
log "Pre-flight checks passed"
}

#-------------------------
# Upgrade check
#-------------------------
upgrade_required() {
CUR_OS=$(ssh "$HOST" oslevel -s | sed 's/-.*//')
log "Current OS: $CUR_OS, Target OS: $TARGET_OS"
[[ "$CUR_OS" -lt "$TARGET_OS" ]]
}

#-------------------------
# Prepare target disk for alt_disk / nimadm
#-------------------------
prepare_target_disk() {
log "Preparing target disk $TARGET_DISK"
ssh "$HOST" bash -s >>"$LOG_FILE" 2>&1 <<EOF
set -o errexit
# Clean existing altinst_rootvg
if lsvg | grep altinst_rootvg >/dev/null 2>&1; then
echo "Cleaning existing altinst_rootvg"
alt_disk_install -X
fi

# Break mirror if disk is part of rootvg
if lspv "$TARGET_DISK" | grep -q rootvg; then
echo "Disk $TARGET_DISK is part of rootvg — breaking mirror"
unmirrorvg rootvg "$TARGET_DISK"
reducevg -df rootvg "$TARGET_DISK"
chpv -c "$TARGET_DISK"
fi

# Rebuild boot info to ensure disk is clean
ipl_varyon -i
bootlist -m normal -o
bosboot -ad "$TARGET_DISK"

# Clear PV attributes
chdev -l "$TARGET_DISK" -a pv=clear
EOF
log "Target disk preparation complete"
}

#-------------------------
# Run nimadm upgrade
#-------------------------
run_nim_upgrade() {
NIM_FLAGS=""
[[ -n "$ALT_DISK_FLAGS" ]] && NIM_FLAGS="-Y $ALT_DISK_FLAGS"
PREVIEW_PARAM=""
[[ "$PREVIEW" -eq 1 ]] && PREVIEW_PARAM="-P"
CMD="nimadm -j $NIMADM_VG -s $SPOT_NAME -l $LPP_NAME -c $HOST -d $TARGET_DISK $PREVIEW_PARAM $NIM_FLAGS 1,2,3,4,5,6,7,8"
log "Executing: $CMD"
eval "$CMD"
}

#-------------------------
# Main workflow
#-------------------------
main() {
check_connectivity
check_nim_client
check_cachevg_space
preflight_checks

if upgrade_required; then
log "Upgrade required"
[[ "$VERIFY" -eq 1 ]] && { log "VERIFY mode — exiting"; exit 0; }
prepare_target_disk
run_nim_upgrade
else
log "No upgrade needed"
fi

log "Upgrade workflow completed successfully"
email_notify "SUCCESS" "$EMAIL_SUBJECT_SUCCESS"
}

main
exit 0

--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
How to Run the Script

1. Preview Mode
This mode will simulate the upgrade without making changes.
# ./aix_os_upgrade-one-host.ksh -o upgrade -d hdisk1 -p my-aix-host

Sample Output:
2026-01-17 10:12:01 : Checking connectivity to my-aix-host
2026-01-17 10:12:02 : Connectivity OK
2026-01-17 10:12:02 : Checking if my-aix-host is a defined NIM client
2026-01-17 10:12:02 : my-aix-host is a valid NIM client
2026-01-17 10:12:03 : Checking client rootvg size and NIM server cachevg free space
2026-01-17 10:12:03 : Client rootvg size: 20480 MB
2026-01-17 10:12:03 : NIM server cachevg free: 51200 MB
2026-01-17 10:12:03 : Running pre-flight checks on my-aix-host
2026-01-17 10:12:04 : Pre-flight checks passed
2026-01-17 10:12:04 : Upgrade required
2026-01-17 10:12:04 : Preparing target disk hdisk1
2026-01-17 10:12:05 : Target disk preparation complete
2026-01-17 10:12:05 : Executing: nimadm -j cachevg -s spot_7300_01_00 -l lpp_7300_01_00 -c my-aix-host -d hdisk1 -P 1,2,3,4,5,6,7,8
2026-01-17 10:12:05 : nimadm preview completed successfully — no changes made
2026-01-17 10:12:05 : Upgrade workflow completed successfully
2026-01-17 10:12:05 : Email notification sent: SUCCESS

2. Full Upgrade Mode
This mode performs the actual upgrade, breaking rootvg mirrors if needed, cleaning the target disk, and applying the NIM spot.
# ./aix_os_upgrade-one-host.ksh -o upgrade -d hdisk1 -f my-aix-host

Sample Output:
2026-01-17 11:00:01 : Checking connectivity to my-aix-host
2026-01-17 11:00:02 : Connectivity OK
2026-01-17 11:00:02 : Checking if my-aix-host is a defined NIM client
2026-01-17 11:00:02 : my-aix-host is a valid NIM client
2026-01-17 11:00:03 : Checking client rootvg size and NIM server cachevg free space
2026-01-17 11:00:03 : Client rootvg size: 20480 MB
2026-01-17 11:00:03 : NIM server cachevg free: 51200 MB
2026-01-17 11:00:03 : Running pre-flight checks on my-aix-host
2026-01-17 11:00:04 : Pre-flight checks passed
2026-01-17 11:00:04 : Upgrade required
2026-01-17 11:00:04 : Preparing target disk hdisk1
2026-01-17 11:00:05 : Disk hdisk1 is part of rootvg — breaking mirror
2026-01-17 11:00:05 : unmirrorvg rootvg hdisk1
2026-01-17 11:00:06 : reducevg -df rootvg hdisk1
2026-01-17 11:00:06 : chpv -c hdisk1
2026-01-17 11:00:07 : ipl_varyon -i
2026-01-17 11:00:07 : bootlist -m normal -o
2026-01-17 11:00:08 : bosboot -ad hdisk1
2026-01-17 11:00:08 : Target disk preparation complete
2026-01-17 11:00:08 : Executing: nimadm -j cachevg -s spot_7300_01_00 -l lpp_7300_01_00 -c my-aix-host -d hdisk1 1,2,3,4,5,6,7,8
2026-01-17 11:30:12 : nimadm upgrade completed successfully
2026-01-17 11:30:12 : Upgrade workflow completed successfully
2026-01-17 11:30:12 : Email notification sent: SUCCESS

Key Notes / Best Practices
  • Always run in preview mode first (-p) to validate disk space, connectivity, and commands.
  • Ensure NIM server has enough cachevg free space before upgrading.
  • The script intelligently handles rootvg mirrors and cleans altinst_rootvg.
  • Logs are saved in /var/log/aix_upgrade/<hostname>.log and email notifications provide audit info.
  • For large rootvg volumes, the script calculates free space using PP sizes, avoiding disk exhaustion.
Conclusion
With this script, upgrading a single AIX host becomes:
  • Safe: avoids accidental rootvg damage
  • Predictable: preview mode validates before actual upgrade
  • Automated: handles mirrors, PV clearing, bootloader, and NIM deployment
  • Auditable: detailed logs and email notifications
This makes your OS upgrade process production-ready and repeatable, reducing downtime and human error.

AIX OS Patching Using Alternate rootvg

This post explains how to safely patch an AIX host using alternate rootvg while handling rootvg mirrors and existing altinst_rootvg. We'll provide a ready-to-run script, workflow diagram, and sample outputs.

Overview
Patching AIX in production can be risky if done on the active rootvg. Using alternate rootvg allows you to:
  • Clone the rootvg to a free disk
  • Apply patches (TL/SP) safely
  • Switch bootlist to the updated rootvg
  • Minimize downtime
The process handles:
  • Rootvg mirror disks
  • Pre-existing altinst_rootvg
  • Disk discovery and NFS mounting for patch repository
Workflow Diagram

Complete Script
The following production-ready script fully automates patching with alternate rootvg, handling mirrors and existing altinst_rootvg.
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

#!/usr/bin/ksh
###############################################################################
# aix_os_patch-one-host.ksh
# Purpose: AIX OS patching using alternate rootvg, handling mirrors and altinst_rootvg
# Author: adminCtrlX 
# Script Preview Mode : ./aix_os_patch-one-host.ksh -h my-aix-host -o patch -p
# Script Full Upgrade : ./aix_os_patch-one-host.ksh -h my-aix-host -o patch -f
###############################################################################
set -o errexit
set -o nounset
set -o pipefail 2>/dev/null || true

###############################################################################
# GLOBALS
###############################################################################
SCRIPT_NAME=$(basename "$0")
BASE_DIR=$(cd "$(dirname "$0")" && pwd)
LOG_DIR="${BASE_DIR}/logs"
SSH="ssh -o BatchMode=yes -o ConnectTimeout=10"

PREVIEW=1
FORCE=0
OPERATIONS=""
TARGET_DISK=""
HOST=""

# NFS / ALT ROOTVG SETTINGS
NFS_SERVER="aixnimserver"
NFS_EXPORT="/exports/software/aix_72.05.10"
NFS_MOUNT="/mnt"
ALT_MOUNT="/alt_inst"

mkdir -p "$LOG_DIR"

###############################################################################
# HELP
###############################################################################
show_help() {
cat <<EOF
Patch One AIX Host (Alternate Rootvg Method)
Usage:
$SCRIPT_NAME -h <hostname> -o patch [options]

Required:
-h hostname Target AIX system
-o patch Perform OS patching

Optional:
-d disk Disk for alternate rootvg
-p Preview mode (default)
-f Force execution
--help Show help

Notes:
* Preview mode makes NO changes
* Force mode performs OS patching
* Reboot is MANUAL after completion
EOF
exit 0
}

###############################################################################
# LOGGING
###############################################################################
log_init() {
    LOG_FILE="${LOG_DIR}/${HOST}.log"
    DRYRUN_FILE="${LOG_DIR}/${HOST}.dryrun.cmds"
    exec > >(tee -a "$LOG_FILE") 2>&1
    : > "$DRYRUN_FILE"
}

log() { print "$(date '+%Y-%m-%d %H:%M:%S') : $*" ; }

fatal() { log "FATAL: $*" ; cleanup ; exit 1 ; }

queue_cmd() { print "$1" >> "$DRYRUN_FILE" ; }

run_cmd() {
    CMD="$1"
    if [[ "$PREVIEW" -eq 1 ]]; then
        log "PREVIEW: $CMD"
        queue_cmd "$CMD"
    else
        log "EXEC: $CMD"
        eval "$CMD"
    fi
}

###############################################################################
# CLEANUP / ROLLBACK
###############################################################################
cleanup() {
    log "Running cleanup"
    run_cmd "$SSH $HOST umount $NFS_MOUNT || true"
    run_cmd "$SSH $HOST alt_root_op -X altinst_rootvg || true"
}
trap cleanup ERR

###############################################################################
# ARGUMENT PARSING
###############################################################################
for arg in "$@"; do
    [[ "$arg" = "--help" ]] && show_help
done

while getopts "h:o:d:pf" opt; do
    case "$opt" in
        h) HOST="$OPTARG" ;;
        o) OPERATIONS="$OPTARG" ;;
        d) TARGET_DISK="$OPTARG" ;;
        p) PREVIEW=1 ;;
        f) PREVIEW=0; FORCE=1 ;;
        *) show_help ;;
    esac
done

[[ -z "$HOST" || "$OPERATIONS" != "patch" ]] && show_help
log_init
log "Starting AIX OS patching for $HOST"

###############################################################################
# PRE-FLIGHT VALIDATION
###############################################################################
[[ "$(id -u)" -ne 0 ]] && fatal "Must be run as root"
ping -c 2 "$HOST" >/dev/null || fatal "Ping failed"
$SSH "$HOST" "true" || fatal "SSH failed"
OS=$($SSH "$HOST" uname -s)
[[ "$OS" != "AIX" ]] && fatal "Target OS is not AIX"
$SSH "$HOST" "command -v alt_disk_copy" >/dev/null || fatal "alt_disk_copy not found"

###############################################################################
# SAFE DISK DISCOVERY
###############################################################################
discover_disk() {
$SSH "$HOST" "
bootdisk=\$(bootinfo -b)
rootsz=\$(lsvg rootvg | awk '/TOTAL PPs/ {print \$3 * \$6}')
lspv | while read d p v; do
    if [[ \"\$v\" = \"None\" && \"\$d\" != \"\$bootdisk\" ]]; then
        size=\$(bootinfo -s \$d)
        [[ \$size -ge \$rootsz ]] && echo \$d
    fi
done | head -1
"
}

[[ -z "$TARGET_DISK" ]] && TARGET_DISK=$(discover_disk)
[[ -z "$TARGET_DISK" ]] && fatal "No suitable free disk found"
log "Target disk selected: $TARGET_DISK"

###############################################################################
# HANDLE ROOTVG UNMIRROR
###############################################################################
handle_rootvg_mirror() {
    log "Checking if $TARGET_DISK is part of rootvg mirror"
    MIRROR=$($SSH "$HOST" "lspv $TARGET_DISK" | awk '/mirror/ {print $1}')
    if [[ -n "$MIRROR" ]]; then
        log "Breaking mirror on $TARGET_DISK"
        run_cmd "$SSH $HOST unmirrorvg rootvg $TARGET_DISK"
        run_cmd "$SSH $HOST reducevg -df rootvg $TARGET_DISK"
        run_cmd "$SSH $HOST chpv -c $TARGET_DISK"
        run_cmd "$SSH $HOST bootlist -m normal -o"
    fi
}

###############################################################################
# CLEANUP EXISTING ALT ROOTVG
###############################################################################
cleanup_alt_rootvg() {
    log "Checking if altinst_rootvg exists on $HOST"
    EXISTS=$($SSH "$HOST" "lsvg altinst_rootvg >/dev/null 2>&1; echo \$?")
    if [[ "$EXISTS" -eq 0 ]]; then
        log "altinst_rootvg exists — removing it before OS patching"
        run_cmd "$SSH $HOST alt_root_op -X altinst_rootvg"
        log "Existing altinst_rootvg removed successfully"
    else
        log "No existing altinst_rootvg found — ready to use $TARGET_DISK"
    fi
}

###############################################################################
# PATCHING STEPS
###############################################################################
mount_nfs_repo() {
    log "Mounting NFS repository"
    run_cmd "$SSH $HOST mkdir -p $NFS_MOUNT"
    run_cmd "$SSH $HOST mount ${NFS_SERVER}:${NFS_EXPORT} $NFS_MOUNT"
}

alt_clone_phase1() {
    log "Creating alternate rootvg (Phase 1)"
    run_cmd "$SSH $HOST alt_disk_copy -d $TARGET_DISK -P1"
}

alt_emgr_commit_lppchk() {
    log "Applying Emergency Fixes (EMGR) to alternate rootvg"

    # Apply all EMGR fixes in priority order
    run_cmd "$SSH $HOST chroot $ALT_MOUNT /usr/sbin/emgr -P"

    log "Checking for applied EMGR packages to remove"
    EMGR_PACKAGES=$($SSH "$HOST" chroot $ALT_MOUNT /usr/sbin/emgr -P | awk '{print $1}' | tail -n +2)
    # Explanation: - awk '{print $1}' → get first column (fix IDs)
    # tail -n +2 → skip header line

    for fix in $EMGR_PACKAGES; do
        log "Removing EMGR package: $fix"
        run_cmd "$SSH $HOST chroot $ALT_MOUNT /usr/sbin/emgr -r -L $fix"
    done

    log "Committing all applied filesets on alternate rootvg"
    run_cmd "$SSH $HOST chroot $ALT_MOUNT /usr/sbin/installp -c all"

    log "Running final LPP check (level 3) on alternate rootvg"
    run_cmd "$SSH $HOST chroot $ALT_MOUNT /usr/bin/lppchk -vm3"
}

alt_clone_phase23() {
    log "Applying TL/SP (Phase 2/3)"
    run_cmd "$SSH $HOST alt_disk_copy -d $TARGET_DISK -P23 -l $NFS_MOUNT -b update_all"
}

verify_alt_rootvg() {
    log "Waking alternate rootvg"
    run_cmd "$SSH $HOST alt_root_op -W -d $TARGET_DISK"
    log "OS level on alternate rootvg"
    run_cmd "$SSH $HOST chroot $ALT_MOUNT /usr/bin/oslevel -s"
    log "Final lppchk"
    run_cmd "$SSH $HOST chroot $ALT_MOUNT /usr/bin/lppchk -vm3"
}

switch_bootlist() {
    log "Switching bootlist to alternate rootvg"
    run_cmd "$SSH $HOST bootlist -m normal $TARGET_DISK"
    run_cmd "$SSH $HOST bootlist -m normal -o"
}

sleep_alt_rootvg() {
    log "Sleeping alternate rootvg"
    run_cmd "$SSH $HOST alt_root_op -S -d $TARGET_DISK"
}

###############################################################################
# EXECUTION
###############################################################################
mount_nfs_repo
handle_rootvg_mirror
cleanup_alt_rootvg
alt_clone_phase1
alt_emgr_commit_lppchk
alt_clone_phase23
verify_alt_rootvg
switch_bootlist
sleep_alt_rootvg
run_cmd "$SSH $HOST umount $NFS_MOUNT"

###############################################################################
# FINAL
###############################################################################
log "AIX OS patching completed successfully for $HOST"
log "Manual reboot required to activate new rootvg"
[[ "$PREVIEW" -eq 1 ]] && log "Dry-run commands saved in: $DRYRUN_FILE"
exit 0

--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

Sample Run — Preview Mode
# ./aix_os_patch-one-host.ksh -h my-aix-host -o patch -p

Output:
2026-01-17 12:00:01 : Starting AIX OS patching for my-aix-host
2026-01-17 12:00:02 : PREVIEW: ssh my-aix-host mkdir -p /mnt
2026-01-17 12:00:02 : PREVIEW: ssh my-aix-host mount aixnimserver:/exports/software/aix_72.05.10 /mnt
2026-01-17 12:00:03 : PREVIEW: ssh my-aix-host alt_disk_copy -d hdisk2 -P1
2026-01-17 12:00:03 : PREVIEW: ssh my-aix-host alt_disk_copy -d hdisk2 -P23 -l /mnt -b update_all
2026-01-17 12:00:04 : PREVIEW: ssh my-aix-host alt_root_op -W -d hdisk2
2026-01-17 12:00:04 : PREVIEW: ssh my-aix-host chroot /alt_inst oslevel -s
2026-01-17 12:00:05 : PREVIEW: ssh my-aix-host chroot /alt_inst lppchk -vm3
2026-01-17 12:00:05 : PREVIEW: ssh my-aix-host bootlist -m normal hdisk2
2026-01-17 12:00:05 : PREVIEW: ssh my-aix-host bootlist -m normal -o
2026-01-17 12:00:06 : PREVIEW: ssh my-aix-host alt_root_op -S -d hdisk2
2026-01-17 12:00:06 : PREVIEW: ssh my-aix-host umount /mnt
2026-01-17 12:00:06 : Dry-run commands saved in: logs/my-aix-host.dryrun.cmds

Sample Run — Full Upgrade
# ./aix_os_patch-one-host.ksh -h my-aix-host -o patch -f

Output:
2026-01-17 12:10:01 : Starting AIX OS patching for my-aix-host
2026-01-17 12:10:02 : EXEC: ssh my-aix-host mkdir -p /mnt
2026-01-17 12:10:02 : EXEC: ssh my-aix-host mount aixnimserver:/exports/software/aix_72.05.10 /mnt
2026-01-17 12:10:03 : EXEC: ssh my-aix-host alt_disk_copy -d hdisk2 -P1
2026-01-17 12:10:10 : EXEC: ssh my-aix-host alt_disk_copy -d hdisk2 -P23 -l /mnt -b update_all
2026-01-17 12:10:20 : EXEC: ssh my-aix-host alt_root_op -W -d hdisk2
2026-01-17 12:10:21 : EXEC: ssh my-aix-host chroot /alt_inst oslevel -s
2026-01-17 12:10:22 : EXEC: ssh my-aix-host chroot /alt_inst lppchk -vm3
2026-01-17 12:10:22 : EXEC: ssh my-aix-host bootlist -m normal hdisk2
2026-01-17 12:10:23 : EXEC: ssh my-aix-host bootlist -m normal -o
2026-01-17 12:10:23 : EXEC: ssh my-aix-host alt_root_op -S -d hdisk2
2026-01-17 12:10:24 : EXEC: ssh my-aix-host umount /mnt
2026-01-17 12:10:24 : AIX OS patching completed successfully for my-aix-host
2026-01-17 12:10:24 : Manual reboot required to activate new rootvg

Conclusion
This production-ready script allows safe, automated AIX OS patching with:
  • Alternate rootvg creation and TL/SP application
  • Rootvg mirror handling
  • Existing altinst_rootvg cleanup
  • NFS repository mounting
  • Preview mode for safe testing before execution
Tip: Always run preview mode first to ensure disk selection and commands are safe.

Automating Linux LVM Provisioning Using a CSV File

Managing LVM creation across multiple Linux servers can quickly become repetitive and error-prone. In this post, we will walk through a simple, production-ready Bash automation that provisions PV, VG, LV, filesystem, mount point, and fstab entries on multiple servers using a CSV file as input.

This approach is ideal for system administrators who want repeatable, auditable, and scalable storage provisioning.

What This Script Does
Using a single CSV file, the script will:
  • Connect to each server listed in the CSV
  • Validate block devices
  • Create Physical Volumes (PV)
  • Create or reuse Volume Groups (VG)
  • Create Logical Volumes (LV)
  • Create filesystems (XFS or ext4) non-interactively
  • Mount the filesystem
  • Persist the mount in `/etc/fstab`
  • Log all actions per server
The script is idempotent, meaning it is safe to re-run.

CSV Input File
The CSV file is the source of truth for all provisioning operations.

File: `linux_lvm.csv`
# server_name,pv_name,vg_name,lv_name,lv_size,lv_type,mount_point
192.168.10.220,/dev/sda,datavg,lv_app,2G,xfs,/app
192.168.10.220,/dev/sda,datavg,lv_dba,2G,xfs,/dba
192.168.10.221,/dev/sda,datavg,lv_app,2G,xfs,/app
192.168.10.221,/dev/sdb,datavg,lv_dba,2G,xfs,/dba

Column Explanation
server_name  ---> Hostname or IP address to SSH into 
pv_name  ---> Block device to use as PV
vg_name  ---> Volume Group name 
lv_name ---> Logical Volume name
lv_size  ---> Size of the LV (example: 2G)
lv_type  ---> Filesystem type (`xfs` or `ext4`) 
mount_point  ---> Mount directory 

Script Overview

The script performs the following steps for each row in the CSV:
1. Reads the CSV (skipping the header)
2. Validates required fields and filesystem type
3. SSHs into the target server
4. Checks if the PV exists, otherwise creates it
5. Checks if the VG exists, otherwise creates it
6. Creates the LV if it does not exist
7. Detects existing filesystem signatures
8. Creates the filesystem safely and non-interactively
9. Mounts the filesystem
10. Adds a persistent `/etc/fstab` entry
11. Logs output to `/var/log/lvm_provision_<hostname>.log`

LVM Provisioning Script
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
#!/bin/bash
# Author: adminCtrlX
# Purpose: LVM provisioning using server_name from CSV

CSV_FILE="/tmp/scripts/linux_lvm.csv"
SUPPORTED_FS="ext4 xfs"

[ ! -f "$CSV_FILE" ] && {
echo "ERROR: CSV file not found: $CSV_FILE"
exit 1
}

tail -n +2 "$CSV_FILE" | while IFS=',' read -r \
server_name pv_name vg_name lv_name lv_size lv_type mount_point
do
echo "========== Processing $server_name =========="

if [[ -z "$server_name" || -z "$pv_name" || -z "$vg_name" || -z "$lv_name" || \
-z "$lv_size" || -z "$lv_type" || -z "$mount_point" ]]; then
echo "ERROR: Missing mandatory CSV fields. Skipping row."
continue
fi

[[ ! "$lv_size" =~ ^[0-9]+G$ ]] && continue
[[ ! " $SUPPORTED_FS " =~ " $lv_type " ]] && continue

ssh "$server_name" sudo bash <<EOF
LOG_FILE="/var/log/lvm_provision_\$(hostname).log"
exec > >(tee -a "\$LOG_FILE") 2>&1

[ ! -b "$pv_name" ] && { echo "Block device not found"; exit 1; }
pvs "$pv_name" &>/dev/null || pvcreate "$pv_name"
vgs "$vg_name" &>/dev/null || vgcreate "$vg_name" "$pv_name"

LV_PATH="/dev/$vg_name/$lv_name"
lvs "\$LV_PATH" &>/dev/null || lvcreate -L "$lv_size" -n "$lv_name" "$vg_name"

FS_EXIST=\$(lsblk -no FSTYPE "\$LV_PATH")
if [ -z "\$FS_EXIST" ]; then
wipefs -a "\$LV_PATH"
mkfs.$lv_type -f "\$LV_PATH"
fi

mkdir -p "$mount_point"
mountpoint -q "$mount_point" || mount "\$LV_PATH" "$mount_point"
grep -q "\$LV_PATH" /etc/fstab || \
echo "\$LV_PATH $mount_point $lv_type defaults 0 0" >> /etc/fstab
echo "SUCCESS on \$(hostname)"
EOF
done
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Logging
Each server logs execution details to:
/var/log/lvm_provision_<hostname>.log
This makes troubleshooting and auditing straightforward.

Benefits of This Approach
  • CSV-driven infrastructure
  • No interactive prompts
  • Safe to re-run
  • Scales to dozens of servers
  • Clear separation of data and logic
  • Ideal for automation and CI/CD pipelines
Final Thoughts
Using a CSV-driven LVM provisioning script gives you consistency, speed, and reliability when managing storage across multiple Linux systems. This method is especially useful in environments where infrastructure must be provisioned repeatedly with minimal human error.
If you’d like to extend this further, consider adding:
  • DRY-RUN mode
  • Filesystem resize support
  • Wipe control per volume
  • Parallel SSH execution

Automating User Creation on Multiple Linux Servers Using Bash and CSV

Managing user accounts across multiple Linux servers can quickly become repetitive and error-prone. In this post, we walk through a Bash-based automation approach that creates users on multiple remote servers using a CSV input file.
This solution is ideal for system administrators who want a simple, SSH-based alternative to heavier tools while still maintaining consistency and control.

Overview
The script:
  • Reads user details from a CSV file
  • Creates primary groups with specific GIDs
  • Adds users to up to three secondary groups
  • Creates users only if they do not already exist
  • Executes the process on multiple remote servers passed as arguments
CSV File Format
The script expects the following CSV structure:
user_id,
user_pri_group,
user_pri_group_id,
user_sec_group1,
user_sec_group2,
user_sec_group3,
user_home_dir,
user_shell,
user_password,
user_gecos_info

Sample CSV Input
# user_id,user_pri_group,user_pri_group_id,user_sec_group1,user_sec_group2,user_sec_group3,user_home_dir,user_shell,user_password,user_gecos_info

tasleem,tasleem,1005,apps,dba,sysadm,/home/tasleem,/bin/bash,root123,Tasleem Ahmed Khan
hamzah,hamzah,1006,apps,dba,sysadm,/home/hamzah,/bin/bash,root123,Hamzah Ali Khan

Primary Group Enforcement
The primary group name and GID must be present. If missing, user creation is skipped.

Secondary Group Handling
Up to three secondary groups are supported.
If a group does not exist, it is created automatically.

Idempotent Execution

If a user already exists, the script safely skips creation.

Multi-Server Support
The same CSV file is applied to all servers passed on the command line.

Bash Script
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
#!/bin/bash
#
# Author : adminCtrlX
# Description : Automating User Creation on Multiple Linux Servers Using Bash and CSV
# Usage : ./create_users_remote.sh host1 host2 host3
#

CSV_FILE="/tmp/scripts/users.csv"

if [ $# -lt 1 ]; then
echo "Usage: $0 host1 host2 ... hostN"
exit 1
fi

HOSTS="$@"

tail -n +2 "$CSV_FILE" | while IFS=',' read -r \
user_id pri_group pri_gid sec_grp1 sec_grp2 sec_grp3 home_dir shell password gecos
do
for server in $HOSTS; do
echo "Processing user $user_id on $server..."

ssh "$server" sudo bash <<EOF

if [ -z "$pri_group" ] || [ -z "$pri_gid" ]; then
echo "Primary group or GID missing for $user_id. Skipping."
exit 0
fi

if ! getent group "$pri_group" >/dev/null; then
groupadd -g "$pri_gid" "$pri_group"
fi

SEC_GROUPS=""
for grp in "$sec_grp1" "$sec_grp2" "$sec_grp3"; do
if [ -n "\$grp" ]; then
getent group "\$grp" >/dev/null || groupadd "\$grp"
SEC_GROUPS="\$SEC_GROUPS,\$grp"
fi
done

SEC_GROUPS="\${SEC_GROUPS#,}"

if ! id "$user_id" >/dev/null 2>&1; then
useradd \
-g "$pri_group" \
\${SEC_GROUPS:+-G "\$SEC_GROUPS"} \
-d "$home_dir" \
-s "$shell" \
-c "$gecos" \
-m "$user_id"

echo "$user_id:$password" | chpasswd
echo "User $user_id created successfully on $server"
else
echo "User $user_id already exists on $server"
fi
EOF
done
done
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

How to Run the Script
1. Make the script executable:
# chmod +x create_users_remote.sh

2. Execute it by passing the target servers:
# ./create_users_remote.sh server1 server2 server3

Security Considerations
Storing plain-text passwords in CSV files is not recommended for production environments. Consider:
  • Using hashed passwords
  • Forcing password change on first login
  • Using SSH keys instead of passwords
Conclusion
This Bash-based approach provides a lightweight yet effective way to manage users across multiple Linux servers. It is easy to understand, easy to modify, and suitable for small to medium-scale environments where full configuration management tools may not be required.

Installing Atlassian Jira Software 8.19.0 on RHEL 10

Atlassian Jira is a widely used issue and project tracking tool. In this post, we’ll walk through the installation of Jira Software 8.19.0 on Red Hat Enterprise Linux (RHEL) 10 using the official Linux installer (.bin). This guide assumes you are logged in as the root user.

Prerequisites
Before starting, ensure:
  • RHEL 10 is installed and running
  • You have root or sudo privileges
  • Required ports are free: 8080 (HTTP) & 8005 (Control/RMI)
  • At least 2 GB RAM (4 GB recommended)
Step 1: Download the Jira Installer
Download the Jira Software binary installer from Atlassian’s official site (https://www.atlassian.com/software/jira/download/data-center) and place it in your working directory (for example, /tmp).
Example file:
atlassian-jira-software-8.19.0-x64.bin

Step 2: Make the Installer Executable
Change the file permissions to make the installer executable:
# chmod 775 atlassian-jira-software-8.19.0-x64.bin
Verify permissions:
# ll
Output:
-rwxrwxr-x 1 root root 462430556 Sep 15 2021 atlassian-jira-software-8.19.0-x64.bin

Step 3: Run the Jira Installer
Start the installation:
# ./atlassian-jira-software-8.19.0-x64.bin
The installer will unpack its bundled JRE and launch the setup wizard.

Step 4: Installer Configuration Options
During installation, select the following options:
Installation Type
Choose:
Custom Install (recommended for advanced users)
Installation Directory
/opt/atlassian/jira
Jira Home (Data Directory)
/var/atlassian/application-data/jira
Ports Configuration
Use default ports:
HTTP Port: 8080
Control (RMI) Port: 8005

Run Jira as a Service
Choose:
Yes
This ensures Jira starts automatically on system reboot.

Step 5: Confirm Installation Settings
Summary displayed by the installer:
Installation Directory: /opt/atlassian/jira
Home Directory: /var/atlassian/application-data/jira
HTTP Port: 8080
RMI Port: 8005
Install as Service: Yes
Proceed by selecting Install.

Step 6: Start Jira Software
Once the installation completes, choose:
Start Jira Software 8.19.0 now? Yes
The installer will start Jira and register it as a system service.

Step 7: Access Jira Web Interface
After startup, Jira becomes accessible at:
http://localhost:8080
When you open this URL in your browser, you will see the initial setup screen with two options:

Choose the option that best fits your environment. and login google account

Next show server IP address Example 192.168.10.120 & confirm 
Next Setup Administrator account then Next 
Continue with English 
The screen will show as below and you can create project & import issue.
Conclusion
You have successfully installed Atlassian Jira Software 8.19.0 on RHEL 10 using the official Linux installer. Jira is now running as a service and ready for initial configuration via the web interface.

This setup is ideal for:
Agile project management
Issue tracking
Enterprise team collaboration

Automating Multi-Host Active Directory Join on Linux

Managing Linux servers in an Active Directory (AD) environment can be tedious, especially when you have multiple hosts that need to join the domain. Doing this manually is time-consuming, error-prone, and inconsistent.
In this post, I’ll share a fully automated, secure, and parallelized Bash script that allows you to join multiple Linux hosts to an AD domain, configure SSSD, home directories, and sudo permissions for AD groups — all in one go.

Why Automation Matters
  • Manually joining Linux hosts to AD can involve:
  • Installing the required packages (realmd, sssd, adcli, etc.)
  • Editing /etc/resolv.conf and /etc/hosts
  • Configuring Kerberos encryption
  • Joining the AD domain using credentials
  • Setting up SSSD and home directory creation
  • Granting sudo privileges to AD groups
Doing this across 10, 50, or 100 hosts is not scalable. With this script, it becomes:
Secure – prompts once for your AD password and never stores it in plain text.
Parallel – all hosts are processed simultaneously for faster deployment.
Robust – logs are kept per host, so troubleshooting is easier.

The Script: Multi-Host AD Join
Below is the script. Save it as ad_join_multi_pw.sh, make it executable with chmod +x ad_join_multi_hosts.sh, and run it with your hostnames as arguments:
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
#!/bin/bash
#
# Author : adminCtrlX
# Description : Multi-host AD join with secure password prompt, parallel execution, logs
# Usage : ./ad_join_multi_hosts.sh host1 host2 host3
#
set -euo pipefail

### CHECK ARGUMENTS ###
if [ $# -lt 1 ]; then
echo "Usage: $0 <hostname1> [hostname2 ...]"
exit 1
fi

HOSTS=("$@")
AD_ADMIN="administrator"
AD_SERVER_FQDN="inddcpads01.ppc.com"
DOMAIN="ppc.com"
REALM="PPC.COM"
AD_GROUP="unix_admin"

# Prompt once for the AD password
read -s -p "Enter AD password for $AD_ADMIN: " AD_PASSWORD
echo
echo "===== Starting AD Integration for ${#HOSTS[@]} host(s) ====="

for TARGET_HOST in "${HOSTS[@]}"; do
(
LOG_FILE="/tmp/ad_join_${TARGET_HOST}.log"
echo "===== Starting AD join on $TARGET_HOST =====" | tee -a "$LOG_FILE"

ssh -q root@"$TARGET_HOST" bash -s "$DOMAIN" "$REALM" "$AD_SERVER_FQDN" "$AD_GROUP" "$AD_ADMIN" "$AD_PASSWORD" >> "$LOG_FILE" 2>&1 <<'REMOTE_EOF'

set -euo pipefail

# Assign positional parameters
DOMAIN="$1"
REALM="$2"
AD_SERVER_FQDN="$3"
AD_GROUP="$4"
AD_ADMIN="$5"
AD_PASSWORD="$6"
RESOLV_CONF="/etc/resolv.conf"
HOSTS_FILE="/etc/hosts"
SSSD_CONF="/etc/sssd/sssd.conf"
KRB_CRYPTO="/etc/krb5.conf.d/crypto-policies"
SUDOERS_FILE="/etc/sudoers.d/unix_admin"

echo "===== Running AD Integration on $(hostname) ====="

# Install required packages
dnf install -y realmd sssd oddjob oddjob-mkhomedir adcli samba-common-tools krb5-workstation

# Configure DNS (append safely)
grep -q "$DOMAIN" "$RESOLV_CONF" || echo "search $DOMAIN" >> "$RESOLV_CONF"
grep -q "192.168.10.100" "$RESOLV_CONF" || echo "nameserver 192.168.10.100" >> "$RESOLV_CONF"
grep -q "192.168.20.100" "$RESOLV_CONF" || echo "nameserver 192.168.20.100" >> "$RESOLV_CONF"

# Update /etc/hosts if missing
grep -q "$AD_SERVER_FQDN" "$HOSTS_FILE" || echo "192.168.10.100 $AD_SERVER_FQDN inddcpads01" >> "$HOSTS_FILE"

# Configure Kerberos crypto policies
mkdir -p "$(dirname "$KRB_CRYPTO")"
cat <<KRB > "$KRB_CRYPTO"
[libdefaults]
permitted_enctypes = aes256-cts-hmac-sha1-96 aes256-cts-hmac-sha384-192 camellia256-cts-cmac aes128-cts-hmac-sha1-96 aes128-cts-hmac-sha256-128 camellia128-cts-cmac rc4-hmac
KRB

# Join AD domain using password
echo "$AD_PASSWORD" | realm join -v -U "$AD_ADMIN" "$AD_SERVER_FQDN"

# Configure SSSD
cat <<SSSD > "$SSSD_CONF"
[sssd]
domains = $DOMAIN
services = nss, pam
[domain/$DOMAIN]
id_provider = ad
ad_domain = $DOMAIN
krb5_realm = $REALM
cache_credentials = True
use_fully_qualified_names = False
fallback_homedir = /home/%u
default_shell = /bin/bash
access_provider = simple
simple_allow_groups = $AD_GROUP
SSSD

chmod 600 "$SSSD_CONF"
systemctl enable --now sssd

# Ensure oddjobd for home directories
systemctl enable --now oddjobd.service
authselect select sssd with-mkhomedir --force

# Configure sudoers for AD group
echo "%$AD_GROUP ALL=(ALL) NOPASSWD: ALL" > "$SUDOERS_FILE"
chmod 440 "$SUDOERS_FILE"
visudo -cf "$SUDOERS_FILE"

# Permit AD group
realm permit -g "$AD_GROUP"
echo "===== AD Integration Completed on $(hostname) ====="
REMOTE_EOF
echo "===== Finished AD join on $TARGET_HOST. Log: $LOG_FILE ====="
) &
done

# Wait for all parallel jobs to finish
wait
unset AD_PASSWORD
echo "===== All AD joins completed ====="
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
How It Works
The script prompts for the AD administrator password once and never stores it.
Each host is joined to AD in parallel, logging progress to /tmp/ad_join_<hostname>.log.
DNS and /etc/hosts are updated safely.
Kerberos encryption is configured to meet AD standards.
SSSD is configured for AD authentication, with automatic home directory creation (oddjobd).
The specified AD group (unix_admin) is granted sudo privileges.

Usage Example
chmod +x ad_join_multi_hosts.sh
./ad_join_multi_hosts.sh host1 host2 host3
Check logs for each host in /tmp/ad_join_<hostname>.log.
After completion, all hosts will be AD-joined, with SSSD running and AD users able to log in with home directories and sudo privileges.

Benefits
Fast – parallel execution saves time.
Secure – password handled safely.
Reproducible – consistent configuration across all hosts.
Auditable – per-host logs help track exactly what happened.

This script is perfect for Linux administrators managing multiple AD-integrated servers in enterprise environments.

Automating Linux Active Directory Integration with a Shell Script

Integrating Linux servers with Microsoft Active Directory (AD) is a standard requirement in enterprise environments. While manual configuration works, it becomes inefficient and error-prone when managing multiple servers.
This post explains how to automate Linux–AD integration using a single Bash script, covering installation, configuration, domain joining, access control, and sudo permissions.

Why Use a Script?
Automating AD integration provides:
  • Consistent configuration across servers
  • Faster provisioning
  • Reduced human error
  • Easy reuse for new environments
  • Improved operational reliability
What the Script Accomplishes
The script performs the following:
  • Installs required AD integration packages
  • Configures DNS resolution
  • Updates /etc/hosts
  • Configures Kerberos encryption policies
  • Enables SSSD and automatic home directory creation
  • Joins the Linux system to Active Directory
  • Restricts login access to a specific AD group
  • Configures SSSD securely
  • Grants sudo access to an AD group
  • Restarts required services
Prerequisites
Before running the script, ensure:
  • You are logged in as root
  • The system can reach the AD Domain Controller
  • Correct DNS and domain information is available
  • The AD group unix_admin exists
  • You have AD administrator credentials
Complete AD Integration Script
Save the following script as ad_join.sh:
-------------------------------------------------------------------------------------------------------------------------------------------------------------------
#!/bin/bash
#  Author        : adminCtrlX
#  Description : Automates Linux Active Directory integration on server.
#                     - Installs required packages
#                     - Configures DNS, Kerberos, and SSSD
#                     - Joins the system to Active Directory
#                     - Restricts access to an AD group
#                     - Configures sudo using /etc/sudoers
#  Usage          : ./ad_join.sh

set -e

### VARIABLES ###
DOMAIN="ppc.com"
REALM="PPC.COM"
AD_SERVER_FQDN="inddcpads01.ppc.com"
AD_SERVER_IP1="192.168.10.100"
AD_SERVER_IP2="192.168.20.100"
AD_GROUP="unix_admin"
AD_ADMIN="administrator"
RESOLV_CONF="/etc/resolv.conf"
HOSTS_FILE="/etc/hosts"
SSSD_CONF="/etc/sssd/sssd.conf"
KRB_CRYPTO="/etc/krb5.conf.d/crypto-policies"

echo "===== Starting AD Integration Setup ====="

### STEP 1: Install Required Packages ###
echo "Installing required packages..."
dnf install -y realmd sssd oddjob oddjob-mkhomedir adcli samba-common-tools krb5-workstation

### STEP 2: Configure DNS ###
echo "Configuring DNS..."
cp $RESOLV_CONF ${RESOLV_CONF}.bak
cat <<EOF > $RESOLV_CONF
search $DOMAIN
nameserver $AD_SERVER_IP1
nameserver $AD_SERVER_IP2
EOF

### STEP 3: Update /etc/hosts ###
echo "Updating /etc/hosts..."
cp $HOSTS_FILE ${HOSTS_FILE}.bak
if ! grep -q "$AD_SERVER_FQDN" $HOSTS_FILE; then
echo "$AD_SERVER_IP1 $AD_SERVER_FQDN inddcpads01" >> $HOSTS_FILE
fi

### STEP 4: Configure Kerberos Encryption Types ###
echo "Configuring Kerberos crypto policies..."
cp $KRB_CRYPTO ${KRB_CRYPTO}.bak || true
cat <<EOF > $KRB_CRYPTO
[libdefaults]
permitted_enctypes = aes256-cts-hmac-sha1-96 aes256-cts-hmac-sha384-192 camellia256-cts-cmac aes128-cts-hmac-sha1-96 aes128-cts-hmac-sha256-128 camellia128-cts-cmac rc4-hmac
EOF

### STEP 5: Configure Authselect and Oddjob ###
echo "Configuring authselect and oddjob..."
mkdir -p /etc/authselect
authselect select sssd with-mkhomedir --force
systemctl enable --now oddjobd.service

### STEP 6: Join AD Domain ###
echo "Joining AD domain..."
realm join -v -U $AD_ADMIN $AD_SERVER_FQDN

### STEP 7: Verify Domain Join ###
echo "Verifying domain join..."
realm list

### STEP 8: Permit AD Group Login ###
echo "Permitting AD group access..."
realm permit -g $AD_GROUP

### STEP 9: Configure SSSD ###
echo "Configuring SSSD..."
cp $SSSD_CONF ${SSSD_CONF}.bak || true
cat <<EOF > $SSSD_CONF
[sssd]
domains = $DOMAIN
services = nss, pam
[domain/$DOMAIN]
ad_server = $AD_SERVER_FQDN
ad_domain = $DOMAIN
krb5_realm = $REALM
realmd_tags = manages-system joined-with-adcli
cache_credentials = True
id_provider = ad
krb5_store_password_if_offline = True
default_shell = /bin/bash
ldap_id_mapping = True
use_fully_qualified_names = False
fallback_homedir = /home/%u
access_provider = simple
simple_allow_groups = $AD_GROUP
EOF

chmod 600 $SSSD_CONF

### STEP 10: Restart SSSD ###
echo "Restarting SSSD..."
systemctl restart sssd

### STEP 11: AD Group/User Creation ###
echo "NOTE: Ensure AD group '$AD_GROUP' exists and users are added on the AD server."

### STEP 12: Configure Sudo Access ###
echo "Configuring sudo access..."
if ! grep -q "%$AD_GROUP" /etc/sudoers; then
echo "%$AD_GROUP ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
fi

### STEP 13: Final Test Instructions ###
echo "===== Setup Complete ====="
echo "Test with:"
echo " ssh sysadm@$(hostname)"
echo " sudo su -"
-------------------------------------------------------------------------------------------------------------------------------------------------------------------
How to Run the Script
chmod +x ad_join.sh
./ad_join.sh
You will be prompted for the Active Directory administrator password during the domain join process.

Testing the Configuration
Log in using an AD user:
ssh sysadm@hostname
Verify sudo/root access:
sudo su -
If successful, the system is fully integrated with Active Directory.

Best Practices & Improvements
  • Use /etc/sudoers.d/unix_admin instead of editing /etc/sudoers
  • Configure DNS via nmcli on NetworkManager systems
  • Use keytabs for non-interactive domain joins
  • Convert this script into an Ansible role
  • Add logging and rollback mechanisms
Conclusion
This script provides a clean, repeatable, and enterprise-ready solution for integrating Linux systems with Active Directory. It is ideal for system administrators, DevOps teams, and automated provisioning workflows.

Join a RHEL Linux Server to Active Directory Using SSSD

This guide explains how to join a Linux server to an Active Directory (AD) domain, configure authentication using SSSD, allow AD groups to log in, and grant sudo privileges to a specific AD group.

Prerequisites
  • RHEL Linux Servers 8/9/10
  • Proper DNS connectivity to the AD server
  • AD administrator credentials
  • Root or sudo access on the Linux server
Step 1: Install Required Packages
Install all necessary packages for AD integration.
# dnf install -y realmd sssd oddjob oddjob-mkhomedir adcli samba-common-tools krb5-workstation

Step 2: Configure DNS Resolution
Edit /etc/resolv.conf
# vi /etc/resolv.conf
Add the following entries:
search ppc.com
nameserver 192.168.10.100
nameserver 192.168.20.100
Save and exit:
Press Esc
Type :wq!
Press Enter

Step 3: Update /etc/hosts

# vi /etc/hosts
Add the AD server entry:
192.168.10.100 inddcpads01.ppc.com inddcpads01
Save and exit using :wq!.

Step 4: Configure Kerberos Encryption Types
Edit the crypto policy file:
# vi /etc/krb5.conf.d/crypto-policies
Add the following:
[libdefaults]
permitted_enctypes = aes256-cts-hmac-sha1-96 aes256-cts-hmac-sha384-192 camellia256-cts-cmac aes128-cts-hmac-sha1-96 aes128-cts-hmac-sha256-128 camellia128-cts-cmac rc4-hmac
Save and exit.

Step 5: Configure Authselect and Oddjob
If you see the message:
Directory [/etc/authselect] does not exist, please create it!
Run the following commands:
# mkdir /etc/authselect
# authselect select sssd with-mkhomedir --force
# systemctl enable --now oddjobd.service
This enables automatic home directory creation for AD users.

Step 6: Join the Linux Server to the AD Domain
Join the system to the domain using an AD administrator account:
# realm join -v -U administrator inddcpads01.ppc.com
Enter the AD Administrator password when prompted.

Step 7: Verify Domain Join Status
Check whether the system successfully joined the domain:
# realm list
Confirm that:
The domain is listed
permitted-groups includes the intended AD group

Step 8: Permit an AD Group to Log In
Allow the AD group unix_admin to access the Linux server:
# realm permit -g unix_admin

Step 9: Configure SSSD
Edit the SSSD configuration file:
# vi /etc/sssd/sssd.conf
Update or add the following configuration:
[domain/ppc.com]
ad_server = inddcpads01.ppc.com
ad_domain = ppc.com
krb5_realm = ppc.com
realmd_tags = manages-system joined-with-adcli
cache_credentials = True
id_provider = ad
krb5_store_password_if_offline = True
default_shell = /bin/bash
ldap_id_mapping = True
use_fully_qualified_names = False
fallback_homedir = /home/%u
access_provider = simple
simple_allow_groups = unix_admin

Save and exit.
Important: Ensure correct file permissions:
# chmod 600 /etc/sssd/sssd.conf

Step 10: Restart SSSD Service
# systemctl restart sssd

Step 11: Create AD Group and Add Users (On AD Server)
Create the AD group: unix_admin
Add required AD users (e.g., sysadm) to this group
Allow time for AD replication if needed

Step 12: Configure Sudo Access for AD Group
Edit the sudoers file safely:
# visudo
Add the following line:
%unix_admin ALL=(ALL) NOPASSWD: ALL
This grants passwordless sudo access to all members of the unix_admin group.

Step 13: Test Login and Root Access

Log in using an AD user (example: sysadm)
Verify sudo access:
$ sudo su -
If successful, the user now has root privileges.

Conclusion
You have successfully:
  • Joined  Linux server to Active Directory
  • Configured SSSD authentication
  • Enabled automatic home directory creation
  • Restricted access to a specific AD group
  • Granted sudo/root privileges to AD users
This setup provides centralized authentication, improved security, and easier user management across Linux servers.

Grafana Enterprise Installation

This document provides step-by-step guidance for installing Grafana Enterprise on RHEL/CentOS, configuring NGINX as a reverse proxy with HTTPS (Let’s Encrypt), and enabling high availability (HA) for Grafana, Prometheus, Alertmanager, and the load balancer.

1. Install Grafana Enterprise
1.1 Install RPM Package
# yum install -y https://dl.grafana.com/grafana-enterprise/release/12.3.1/grafana-enterprise_12.3.1_20271043721_linux_amd64.rpm
# OR
# dnf install -y https://dl.grafana.com/grafana-enterprise/release/12.3.1/grafana-enterprise_12.3.1_20271043721_linux_amd64.rpm
1.2 Enable and Start Grafana
# systemctl enable --now grafana-server

2. Configure Grafana Server
2.1 Update Grafana Configuration
# vi /etc/grafana/grafana.ini
Edit [server] section:
[server]
http_addr = localhost
http_port = 3000
domain = www.grafana.ppc.com
Binding Grafana to localhost ensures external access only via NGINX.
2.2 Restart Grafana
# systemctl restart grafana-server

3. Install and Configure NGINX Reverse Proxy
3.1 Install NGINX
# dnf install nginx -y
3.2 NGINX Configuration for Grafana

Create /etc/nginx/conf.d/grafana.conf:

# Proxy WebSocket connections for Grafana Live
map $http_upgrade $connection_upgrade {
  default upgrade;
  '' close;
}
server {
    listen 80;
    server_name grafana.example.io;
    rewrite ^ https://$server_name$request_uri? permanent;
}
server {
  listen 443 ssl http2;
  server_name grafana.example.io;

  root /usr/share/nginx/html;
  index index.html index.htm;

  ssl_certificate /etc/letsencrypt/live/grafana.example.io/fullchain.pem;
  ssl_certificate_key /etc/letsencrypt/live/grafana.example.io/privkey.pem;

  access_log /var/log/nginx/grafana-access.log;
  error_log /var/log/nginx/grafana-error.log;

  location / {
    proxy_pass https://localhost:3000/;
  }

  location /api/live {
    rewrite ^/(.*) /$1 break;
    proxy_http_version 1.1;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection $connection_upgrade;
    proxy_set_header Host $http_host;
    proxy_pass https://localhost:3000/;
  }
}
3.3 Validate and Start NGINX
# nginx -t
# systemctl enable --now nginx
# systemctl status nginx

4. Access Grafana
Open browser at:
https://www.grafana.ppc.com
Default credentials:
Username: admin
Password: admin (change on first login)

5. SELinux Configuration for Grafana + NGINX
5.1 Verify SELinux Status
# getenforce
Expected: Enforcing
5.2 Allow NGINX Network Connections
# setsebool -P httpd_can_network_connect 1
5.3 WebSocket Support
WebSockets for Grafana Live are allowed via the same boolean above. No additional SELinux policies required.
5.4 Optional: Verify Grafana Port Context
# semanage port -l | grep 3000
# semanage port -a -t http_port_t -p tcp 3000  # if required
5.5 Check SELinux Denials
# ausearch -m AVC -ts recent
# journalctl -t setroubleshoot

6. Grafana High Availability (HA)
6.1 HA Requirements
Component Requirement
Database PostgreSQL or MySQL (not SQLite)
Sessions Shared DB
Storage Local disks
Load Balancer NGINX or HAProxy
Grafana Version Enterprise / OSS
6.2 Shared Database Configuration
[database]
type = postgres
host = dbserver.example.io:5432
name = grafana
user = grafana
password = strongpassword
ssl_mode = disable
Restart Grafana on each node:
# systemctl restart grafana-server
6.3 Node Configuration
[server]
http_addr = 0.0.0.0
http_port = 3000
domain = grafana.example.io

[unified_alerting]
enabled = true

Each node listens locally; external access only via the load balancer.

7. Load Balancer Configuration
Option A: NGINX

Create /etc/nginx/conf.d/grafana-ha.conf:

upstream grafana_backend {
    least_conn;
    server indrxgraf01:3000;
    server indrxgraf02:3000;
}

server {
    listen 443 ssl http2;
    server_name grafana.example.io;

    ssl_certificate /etc/letsencrypt/live/grafana.example.io/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/grafana.example.io/privkey.pem;

    location / {
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_pass http://grafana_backend;
    }

    location /api/live {
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
        proxy_set_header Host $host;
        proxy_pass http://grafana_backend;
    }
}
# nginx -t
# systemctl reload nginx
Option B: HAProxy (Alternative)
frontend grafana_https
    bind *:443 ssl crt /etc/haproxy/certs/grafana.pem
    default_backend grafana_nodes

backend grafana_nodes
    balance roundrobin
    server graf01 indrxgraf01:3000 check
    server graf02 indrxgraf02:3000 check

8. Keepalived Load Balancer HA
8.1 Install Keepalived
# dnf install keepalived -y
# echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf
# sysctl -p
8.2 Configure MASTER Node

Edit /etc/keepalived/keepalived.conf:

vrrp_script chk_nginx {
    script "/usr/bin/pidof nginx"
    interval 2
    weight -20
}

vrrp_instance VI_GRAFANA {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 101
    advert_int 1
    authentication { auth_type PASS auth_pass StrongPass }
    virtual_ipaddress { 192.168.20.50/24 }
    track_script { chk_nginx }
}
8.3 Configure BACKUP Node
Same file, change:
state BACKUP
priority 100
8.4 Enable Keepalived
# systemctl enable --now keepalived
# systemctl status keepalived
# ip a | grep 192.168.20.50
8.5 SELinux & Firewall
# setsebool -P keepalived_connect_any 1
# firewall-cmd --add-service=keepalived --permanent
# firewall-cmd --reload

9. Prometheus High Availability
9.1 Install Prometheus
# dnf install prometheus -y
# systemctl enable --now prometheus
9.2 Prometheus Configuration
Edit /etc/prometheus/prometheus.yml:

global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'grafana'
    static_configs:
      - targets:
          - indrxgraf01:3000
          - indrxgraf02:3000
Restart Prometheus:
# systemctl restart prometheus

10. Alertmanager High Availability
10.1 Install Alertmanager
# dnf install alertmanager -y
# systemctl enable --now alertmanager
10.2 Configure Clustering
Node 1:
ALERTMANAGER_OPTS="--cluster.listen-address=0.0.0.0:9094 --cluster.peer=alert01.example.io:9094"

Node 2:
ALERTMANAGER_OPTS="--cluster.listen-address=0.0.0.0:9094 --cluster.peer=alert02.example.io:9094"
10.3 Prometheus Alertmanager Integration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - alert01.example.io:9093
            - alert02.example.io:9093
Restart Prometheus:
# systemctl restart prometheus

11. Grafana Integration
Add Prometheus HA endpoints as data sources
Add Alertmanager HA endpoints for alerting
Grafana handles failover automatically

12. Result
Secure Grafana Enterprise installation with HTTPS
NGINX handles SSL termination and WebSocket connections
Active-active HA for Grafana, Prometheus, Alertmanager, and load balancer
SELinux remains enforcing
No single point of failure

DB2 GPFS Cluster LUN Setup Using iSCSI and LVM on RHEL/CentOS

End-to-End High Availability Storage & Database Architecture

IBM DB2 high availability deployments require consistent shared storage, predictable fencing, and cluster-aware filesystems. When SAN infrastructure is unavailable or cost-prohibitive, iSCSI + GPFS (IBM Spectrum Scale) provides a fully supported, enterprise-grade alternative.

This guide covers the complete stack:
  • iSCSI shared LUNs (targetcli + LVM)
  • Multipath-secured access
  • GPFS cluster installation and quorum design
  • DB2 installation on GPFS
  • Performance tuning and HA validation
Reference Architecture
┌───────────────────────────────┐
│  iSCSI Target Server          │
│  RHEL/CentOS                  │
│  LVM VG: datavg               │
│  ├─ db2disk01 (3G)  → Data    │
│  ├─ db2disk02 (7G)  → Indexes │
│  ├─ db2disk03 (3G)  → Logs    │
│  └─ db2disk04 (7G)  → Temp    │
│  targetcli / LIO              │
└──────────────┬────────────────┘
               │ iSCSI (TCP 3260, MTU 9000)
 ┌─────────────┴─────────────┐
 │                           │
┌────────────────────┐   ┌────────────────────┐
│ DB2 Node 1         │   │ DB2 Node 2         │
indrxldb201        │   │ indrxldb202        │
│ GPFS Node          │   │ GPFS Node          │
│ DB2 Instance       │   │ DB2 Instance       │
└────────────────────┘   └────────────────────┘

Why GPFS for DB2?
RequirementGPFS Capability
Concurrent accessDistributed lock manager
HA fencingQuorum + tiebreaker disks
Write orderingJournaled metadata
Fast failoverSub-30s remount
IBM supportFully certified

ext4/XFS are NOT supported for DB2 HA shared-disk clusters

Environment Summary
ComponentValue
OSRHEL 8/9 or CentOS Stream
GPFSIBM Spectrum Scale 5.1+
DB211.5.x
Network10Gb iSCSI VLAN (MTU 9000)
Cluster Nodesindrxldb201, indrxldb202
QuorumTiebreaker disk required

Part 1 – GPFS Installation (Both Nodes)

1.1 Prerequisites
# dnf install -y \
  kernel-devel \
  gcc \
  cpp \
  elfutils-libelf-devel \
  numactl \
  chrony \
  net-tools
Ensure time synchronization:
# systemctl enable --now chronyd

1.2 Install IBM Spectrum Scale Packages
Upload Spectrum Scale RPMs to both nodes.
# rpm -ivh gpfs.base*.rpm gpfs.gpl*.rpm gpfs.msg*.rpm
Build kernel module:
/usr/lpp/mmfs/bin/mmbuildgpl
Verify:
# lsmod | grep mmfs
# echo "export PATH=$PATH:/usr/lpp/mmfs/bin" >> /root/.bash_profile

1.3 Create GPFS Cluster
Run once from primary node:
# mmcrcluster -N indrxldb201:manager-quorum,indrxldb202:manager-quorum -p indrxldb201 -s indrxldb202 -r /usr/bin/ssh -R /usr/bin/scp -C GPFS_DB2

# /usr/lpp/mmfs/bin/mmstartup -N indrxldb201
# /usr/lpp/mmfs/bin/mmstartup -N indrxldb201

Start GPFS:
# mmstartup -a
Verify:
# mmgetstate -a

1.4 Configure Quorum & Tiebreaker
For 2-node clusters, quorum is mandatory.
# mmadddisk tb -F /dev/mapper/mpatha
# mmchconfig tiebreakerDisks=mpatha
# mmchconfig quorum=4

Vote SourceCount
Node 11
Node 21
Data disk1
Tiebreaker1

Part 2 – GPFS Filesystem Design for DB2
FilesystemPurpose
gpfs_db2dataTablespaces
gpfs_db2indexIndexes
gpfs_db2logsTransaction logs
gpfs_db2tempTemp tables
Create File /tmp/nsdlist
# vi /tmp/nsdlist
%nsd:
device=/dev/sdb
nsd=nsd_01
servers=indrxldb201,indrxldb202
usage=dataAndMetadata 
failureGroup=-1 
pool=system 

Create NSDs
# mmcrnsd -F /tmp/nsdlist
Create Filesystems
# mmcrfs gpfs_db2data  -F /dev/mapper/mpathb -A yes -Q yes
# mmcrfs gpfs_db2index -F /dev/mapper/mpathc -A yes -Q yes
# mmcrfs gpfs_db2logs  -F /dev/mapper/mpathd -A yes -Q no
Mount:
# mmmount all -a

Part 3 – DB2 Installation (Both Nodes)

3.1 OS Kernel Tuning
# sysctl -w kernel.shmmni=8192
# sysctl -w kernel.shmmax=$(($(getconf _PHYS_PAGES) * 4096))
# sysctl -w kernel.shmall=$(($(getconf _PHYS_PAGES) * 4096 / 4096))
# sysctl -w vm.swappiness=1
Persist in /etc/sysctl.conf.

3.2 Create DB2 User & Groups
# groupadd -g 1001 db2iadm1
# groupadd -g 1002 db2fadm1
# useradd -u 1001 -g db2iadm1 -G db2fadm1 db2inst1
# passwd db2inst1

3.3 Install DB2 Software
# tar -xvf DB2_Svr_11.5_Linux_x86-64.tar.gz
# cd server_dec
# ./db2_install
Select:
Server Edition
Typical install

3.4 Create DB2 Instance
# /opt/ibm/db2/V11.5/instance/db2icrt \
  -u db2fadm1 db2inst1

Part 4 – DB2 Database on GPFS
Directory Structure
# mkdir -p /gpfs_db2data/db2inst1
# mkdir -p /gpfs_db2logs/db2inst1
# chown -R db2inst1:db2iadm1 /gpfs*

Create Database
# su - db2inst1
db2 create database PRODDB \
  on /gpfs_db2data \
  dbpath on /gpfs_db2data \
  using codeset UTF-8 territory us

Move logs:
# db2 update db cfg for PRODDB using NEWLOGPATH /gpfs_db2logs/db2inst1

Part 5 – High Availability Behavior
EventResult
Node crashGPFS remounts
Network splitQuorum prevents corruption
iSCSI path lossMultipath reroutes
Storage restartDB2 recovers logs

Typical DB2 recovery time: 20–45 seconds

Performance Tuning
GPFS
# mmchconfig pagepool=80%RAM
# mmchconfig maxFilesToCache=1000000
DB2
# db2 update db cfg using LOGFILSIZ 8192
# db2 update db cfg using LOGPRIMARY 20
# db2 update db cfg using LOGSECOND 100
iSCSI
# echo 256 > /sys/block/sdX/queue/nr_requests

Validation Checklist
  • iSCSI sessions persistent
  • Multipath active
  • GPFS quorum healthy
  • DB2 database starts on either node
  • Forced node failure recovers cleanly
  • No GPFS fencing events
Operational Commands Cheat Sheet
# mmgetstate -a
# mmlscluster
# mmlsdisk gpfs_db2data
# db2pd -db PRODDB -logs
# iscsiadm -m session -P 3
# multipath -ll

Final Thoughts
This architecture delivers:
  • IBM-supported DB2 HA
  • SAN-like behavior using software-defined storage
  • Strong fencing and split-brain prevention
  • Predictable performance at scale
The success factors are discipline and testing:
  • Quorum
  • Multipath
  • Dedicated network
  • Regular failover drills