Difference between revisions of "Xen1 Setup Process"

From CSLabsWiki
m (Configured Additional Xen Bridges)
m (Configured Additional Xen Bridges)
Line 272: Line 272:
 
OP=$1
 
OP=$1
 
shift
 
shift
script=/etc/xen/scripts/network-bridge.xen
+
script=/etc/xen/scripts/network-bridge-cslabs
 
case ${OP} in
 
case ${OP} in
 
start)
 
start)
Line 297: Line 297:
 
*Made script executable
 
*Made script executable
 
**<code>chmod +x /etc/xen/scripts/networks-cslabs</code>
 
**<code>chmod +x /etc/xen/scripts/networks-cslabs</code>
  +
  +
*Copied and modified <code>/etc/xen/scripts/network-bridge</code> script to prevent <code>peth0: received packet with own address as source address</code> errors due to MAC address conflicts between dom0s residing on the same network. (Fix created based on patch submitted by lab alum Cyrus Katrak - [http://bugzilla.xensource.com/bugzilla/show_bug.cgi?id=339 Original Bug Report]
  +
**<code>cp <code>/etc/xen/scripts/network-bridge</code> <code>/etc/xen/scripts/network-bridge-cslabs</code>
  +
<code><pre>
  +
#!/bin/sh
  +
#============================================================================
  +
# Default Xen network start/stop script.
  +
# Xend calls a network script when it starts.
  +
# The script name to use is defined in /etc/xen/xend-config.sxp
  +
# in the network-script field.
  +
#
  +
# This script creates a bridge (default xenbr${vifnum}), adds a device
  +
# (default eth${vifnum}) to it, copies the IP addresses from the device
  +
# to the bridge and adjusts the routes accordingly.
  +
#
  +
# If all goes well, this should ensure that networking stays up.
  +
# However, some configurations are upset by this, especially
  +
# NFS roots. If the bridged setup does not meet your needs,
  +
# configure a different script, for example using routing instead.
  +
#
  +
# Usage:
  +
#
  +
# network-bridge (start|stop|status) {VAR=VAL}*
  +
#
  +
# Vars:
  +
#
  +
# vifnum Virtual device number to use (default 0). Numbers >=8
  +
# require the netback driver to have nloopbacks set to a
  +
# higher value than its default of 8.
  +
# bridge The bridge to use (default xenbr${vifnum}).
  +
# netdev The interface to add to the bridge (default eth${vifnum}).
  +
# antispoof Whether to use iptables to prevent spoofing (default no).
  +
#
  +
# Internal Vars:
  +
# pdev="p${netdev}"
  +
# vdev="veth${vifnum}"
  +
# vif0="vif0.${vifnum}"
  +
#
  +
# start:
  +
# Creates the bridge
  +
# Copies the IP and MAC addresses from netdev to vdev
  +
# Renames netdev to be pdev
  +
# Renames vdev to be netdev
  +
# Enslaves pdev, vdev to bridge
  +
#
  +
# stop:
  +
# Removes netdev from the bridge
  +
# Transfers addresses, routes from netdev to pdev
  +
# Renames netdev to vdev
  +
# Renames pdev to netdev
  +
# Deletes bridge
  +
#
  +
# status:
  +
# Print addresses, interfaces, routes
  +
#
  +
#============================================================================
  +
  +
#macid is used to uniquely identify this dom0 on this network
  +
#change this to avoid MAC address conflicts if you get:
  +
#"peth0: received packet with own address as source address"
  +
macid="E0"
  +
  +
dir=$(dirname "$0")
  +
. "$dir/xen-script-common.sh"
  +
. "$dir/xen-network-common.sh"
  +
  +
findCommand "$@"
  +
evalVariables "$@"
  +
  +
vifnum=${vifnum:-$(ip route list | awk '/^default / { print $NF }' | sed 's/^[^0-9]*//')}
  +
vifnum=${vifnum:-0}
  +
bridge=${bridge:-xenbr${vifnum}}
  +
netdev=${netdev:-eth${vifnum}}
  +
antispoof=${antispoof:-no}
  +
  +
pdev="p${netdev}"
  +
vdev="veth${vifnum}"
  +
vif0="vif0.${vifnum}"
  +
addr_pfx=
  +
  +
get_ip_info() {
  +
addr_pfx=`ip addr show dev $1 | sed -n 's/^ *inet \(.*\) [^ ]*$/\1/p'`
  +
gateway=`ip route show dev $1 | fgrep default | sed 's/default via //'`
  +
}
  +
  +
is_bonding() {
  +
[ -f "/sys/class/net/$1/bonding/slaves" ]
  +
}
  +
  +
is_vlan() {
  +
[ -f "/proc/net/vlan/$1" ]
  +
}
  +
  +
is_ifup() {
  +
ip link show dev $1 | awk '{ exit $3 !~ /[<,]UP[,>]/ }'
  +
}
  +
  +
do_ifup() {
  +
if ! ifup $1 || ! is_ifup $1 ; then
  +
if [ -n "${addr_pfx}" ] ; then
  +
# use the info from get_ip_info()
  +
ip addr flush $1
  +
ip addr add ${addr_pfx} dev $1
  +
ip link set dev $1 up
  +
[ ${gateway} ] && ip route add default via ${gateway}
  +
fi
  +
fi
  +
}
  +
  +
# Usage: transfer_addrs src dst
  +
# Copy all IP addresses (including aliases) from device $src to device $dst.
  +
transfer_addrs () {
  +
local src=$1
  +
local dst=$2
  +
# Don't bother if $dst already has IP addresses.
  +
if ip addr show dev ${dst} | egrep -q '^ *inet ' ; then
  +
return
  +
fi
  +
# Address lines start with 'inet' and have the device in them.
  +
# Replace 'inet' with 'ip addr add' and change the device name $src
  +
# to 'dev $src'.
  +
ip addr show dev ${src} | egrep '^ *inet ' | sed -e "
  +
s/inet/ip addr add/
  +
s@\([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+/[0-9]\+\)@\1@
  +
s/${src}/dev ${dst} label ${dst}/
  +
s/secondary//
  +
" | sh -e
  +
# Remove automatic routes on destination device
  +
ip route list | sed -ne "
  +
/dev ${dst}\( \|$\)/ {
  +
s/^/ip route del /
  +
p
  +
}" | sh -e
  +
}
  +
  +
# Usage: transfer_routes src dst
  +
# Get all IP routes to device $src, delete them, and
  +
# add the same routes to device $dst.
  +
# The original routes have to be deleted, otherwise adding them
  +
# for $dst fails (duplicate routes).
  +
transfer_routes () {
  +
local src=$1
  +
local dst=$2
  +
# List all routes and grep the ones with $src in.
  +
# Stick 'ip route del' on the front to delete.
  +
# Change $src to $dst and use 'ip route add' to add.
  +
ip route list | sed -ne "
  +
/dev ${src}\( \|$\)/ {
  +
h
  +
s/^/ip route del /
  +
P
  +
g
  +
s/${src}/${dst}/
  +
s/^/ip route add /
  +
P
  +
d
  +
}" | sh -e
  +
}
  +
  +
  +
##
  +
# link_exists interface
  +
#
  +
# Returns 0 if the interface named exists (whether up or down), 1 otherwise.
  +
#
  +
link_exists()
  +
{
  +
if ip link show "$1" >/dev/null 2>/dev/null
  +
then
  +
return 0
  +
else
  +
return 1
  +
fi
  +
}
  +
  +
# Set the default forwarding policy for $dev to drop.
  +
# Allow forwarding to the bridge.
  +
antispoofing () {
  +
iptables -P FORWARD DROP
  +
iptables -F FORWARD
  +
iptables -A FORWARD -m physdev --physdev-in ${pdev} -j ACCEPT
  +
iptables -A FORWARD -m physdev --physdev-in ${vif0} -j ACCEPT
  +
}
  +
  +
# Usage: show_status dev bridge
  +
# Print ifconfig and routes.
  +
show_status () {
  +
local dev=$1
  +
local bridge=$2
  +
  +
echo '============================================================'
  +
ip addr show ${dev}
  +
ip addr show ${bridge}
  +
echo ' '
  +
brctl show ${bridge}
  +
echo ' '
  +
ip route list
  +
echo ' '
  +
route -n
  +
echo '============================================================'
  +
}
  +
  +
is_network_root () {
  +
local rootfs=$(awk '{ if ($1 !~ /^[ \t]*#/ && $2 == "/") { print $3; }}' /etc/mtab)
  +
local rootopts=$(awk '{ if ($1 !~ /^[ \t]*#/ && $2 == "/") { print $4; }}' /etc/mtab)
  +
  +
[[ "$rootfs" =~ "^nfs" ]] || [[ "$rootopts" =~ "_netdev" ]] && return 0 || return 1
  +
}
  +
  +
op_start () {
  +
if [ "${bridge}" = "null" ] ; then
  +
return
  +
fi
  +
  +
if is_network_root ; then
  +
[ -x /usr/bin/logger ] && /usr/bin/logger "network-bridge: bridging not supported on network root; not starting"
  +
return
  +
fi
  +
  +
if ! link_exists "$vdev"; then
  +
if link_exists "$pdev"; then
  +
# The device is already up.
  +
return
  +
else
  +
echo "
  +
Link $vdev is missing.
  +
This may be because you have reached the limit of the number of interfaces
  +
that the loopback driver supports. If the loopback driver is a module, you
  +
may raise this limit by passing it as a parameter (nloopbacks=<N>); if the
  +
driver is compiled statically into the kernel, then you may set the parameter
  +
using loopback.nloopbacks=<N> on the domain 0 kernel command line.
  +
" >&2
  +
exit 1
  +
fi
  +
fi
  +
  +
create_bridge ${bridge}
  +
  +
if link_exists "$vdev"; then
  +
mac=`ip link show ${netdev} | grep 'link\/ether' | sed -e 's/.*ether \(..:..:..:..:..:..\).*/\1/'`
  +
preiftransfer ${netdev}
  +
transfer_addrs ${netdev} ${vdev}
  +
if is_bonding ${netdev} || is_vlan ${netdev} || ! ifdown ${netdev}; then
  +
# Remember the IP details if necessary.
  +
get_ip_info ${netdev}
  +
ip link set ${netdev} down
  +
ip addr flush ${netdev}
  +
fi
  +
ip link set ${netdev} name ${pdev}
  +
ip link set ${vdev} name ${netdev}
  +
  +
setup_bridge_port ${pdev}
  +
setup_bridge_port ${vif0}
  +
ip link set ${netdev} addr ${mac} arp on
  +
  +
ip link set ${pdev} addr fe:ff:ff:ff:${macid}:0${vifnum}
  +
ip link set ${vif0} addr fe:ff:ff:ff:${macid}:0${vifnum}
  +
  +
ip link set ${bridge} up
  +
add_to_bridge ${bridge} ${vif0}
  +
add_to_bridge2 ${bridge} ${pdev}
  +
do_ifup ${netdev}
  +
else
  +
# old style without ${vdev}
  +
transfer_addrs ${netdev} ${bridge}
  +
transfer_routes ${netdev} ${bridge}
  +
fi
  +
  +
if [ ${antispoof} = 'yes' ] ; then
  +
antispoofing
  +
fi
  +
}
  +
  +
op_stop () {
  +
if [ "${bridge}" = "null" ]; then
  +
return
  +
fi
  +
if ! link_exists "$bridge"; then
  +
return
  +
fi
  +
  +
if link_exists "$pdev"; then
  +
ip link set dev ${vif0} down
  +
mac=`ip link show ${netdev} | grep 'link\/ether' | sed -e 's/.*ether \(..:..:..:..:..:..\).*/\1/'`
  +
transfer_addrs ${netdev} ${pdev}
  +
if ! ifdown ${netdev}; then
  +
get_ip_info ${netdev}
  +
fi
  +
ip link set ${netdev} down arp off
  +
ip link set ${netdev} addr fe:ff:ff:ff:ff:ff
  +
ip link set ${pdev} down
  +
ip addr flush ${netdev}
  +
ip link set ${pdev} addr ${mac} arp on
  +
  +
brctl delif ${bridge} ${pdev}
  +
brctl delif ${bridge} ${vif0}
  +
ip link set ${bridge} down
  +
  +
ip link set ${netdev} name ${vdev}
  +
ip link set ${pdev} name ${netdev}
  +
do_ifup ${netdev}
  +
else
  +
transfer_routes ${bridge} ${netdev}
  +
ip link set ${bridge} down
  +
fi
  +
brctl delbr ${bridge}
  +
}
  +
  +
# adds $dev to $bridge but waits for $dev to be in running state first
  +
add_to_bridge2() {
  +
local bridge=$1
  +
local dev=$2
  +
local maxtries=10
  +
  +
echo -n "Waiting for ${dev} to negotiate link."
  +
ip link set ${dev} up
  +
for i in `seq ${maxtries}` ; do
  +
if ifconfig ${dev} | grep -q RUNNING ; then
  +
break
  +
else
  +
echo -n '.'
  +
sleep 1
  +
fi
  +
done
  +
  +
if [ ${i} -eq ${maxtries} ] ; then echo '(link isnt in running state)' ; fi
  +
  +
add_to_bridge ${bridge} ${dev}
  +
}
  +
  +
case "$command" in
  +
start)
  +
op_start
  +
;;
  +
  +
stop)
  +
op_stop
  +
;;
  +
  +
status)
  +
show_status ${netdev} ${bridge}
  +
;;
  +
  +
*)
  +
echo "Unknown command: $command" >&2
  +
echo 'Valid commands are: start, stop, status' >&2
  +
exit 1
  +
esac
  +
</pre></code>
   
 
*Edited <code>/etc/xen/xend-config.sxp</code>
 
*Edited <code>/etc/xen/xend-config.sxp</code>

Revision as of 13:30, 28 May 2009

This page summarizes how Xen1 was set up in Spring 2009.

Install

  • Installed CentOS 5.3 x64 with Virtualization Support.
    • Partition Scheme
      • 100 MB /boot - Software RAID 1
      • 73 GB root_lvg - Logical Volume Group Software RAID 1
        • 59 GB / (root_lvg-root_lv)
        • 4 GB /var (root_lvg-var_lv)
        • 10 GB swap (root_lvg-swap_lv)
      • 452 GB /xen - Software RAID 1

Kickstart File

# Kickstart file automatically generated by anaconda.

install
cdrom
lang en_US.UTF-8
keyboard us
network --device eth0 --bootproto static --ip 128.153.145.41 --netmask 255.255.255.0 --gateway 128.153.145.1 --nameserver 128.153.0.254,128.153.5.254 --hostname xen1.cslabs.clarkson.edu
network --device eth1 --bootproto static --ip 10.0.1.37 --netmask 255.255.255.0 --gateway 128.153.145.1 --nameserver 128.153.0.254,128.153.5.254 --hostname xen1.cslabs.clarkson.edu
network --device eth2 --bootproto static --ip 10.0.0.16 --netmask 255.255.255.0 --gateway 128.153.145.1 --nameserver 128.153.0.254,128.153.5.254 --hostname xen1.cslabs.clarkson.edu
rootpw --iscrypted ENCRYPTED-PASSWORD-GOES-HERE
firewall --enabled --port=22:tcp
authconfig --enableshadow --enablemd5
selinux --enforcing
timezone --utc America/New_York
bootloader --location=partition --driveorder=sda,sdb,hda,hdb --md5pass=ENCRYPTED-PASSWORD-GOES-HERE
# The following is the partition information you requested
# Note that any partitions you deleted are not expressed
# here so unless you clear all partitions first, this is
# not guaranteed to work
clearpart --linux
part raid.16 --size=100 --ondisk=sda
part raid.19 --size=100 --ondisk=sdb
part raid.20 --size=100 --grow --ondisk=hdb
part raid.18 --size=100 --grow --ondisk=sdb
part raid.17 --size=100 --grow --ondisk=sda
part raid.14 --size=100 --grow --ondisk=hda
raid /boot --fstype ext3 --level=RAID1 --device=md0 raid.16 raid.19
raid pv.22 --fstype "physical volume (LVM)" --level=RAID1 --device=md1 raid.17 raid.18
raid /xen --fstype ext3 --level=RAID1 --device=md2 raid.14 raid.20
volgroup root_lvg --pesize=32768 pv.22
logvol / --fstype ext3 --name=root_lv --vgname=root_lvg --size=61856
logvol swap --fstype swap --name=swap_lv --vgname=root_lvg --size=10240
logvol /var --fstype ext3 --name=var_lv --vgname=root_lvg --size=4096

%packages
@virtualization
@core
@base
bridge-utils
device-mapper-multipath
-gnome-applet-vm
-NetworkManager
-bluez-utils

Configuration

Updated System

  • Configured Yum ProtectBase & to use our mirror
    • Edited /etc/yum.repos.d/CentOS-Base.repo
# CentOS-Base.repo
#
# This file uses a new mirrorlist system developed by Lance Davis for CentOS.
# The mirror system uses the connecting IP address of the client and the
# update status of each mirror to pick mirrors that are updated to and
# geographically close to the client.  You should use this for CentOS updates
# unless you are manually picking other mirrors.
#
# If the mirrorlist= does not work for you, as a fall back you can try the
# remarked out baseurl= line instead.
#
#

[base]
name=CentOS-$releasever - Base
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os
baseurl=http://mirror.clarkson.edu/centos/$releasever/os/$basearch/
gpgcheck=1
protect=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5

#released updates
[updates]
name=CentOS-$releasever - Updates
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates
baseurl=http://mirror.clarkson.edu/centos/$releasever/updates/$basearch/
gpgcheck=1
protect=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5

#packages used/produced in the build but not released
[addons]
name=CentOS-$releasever - Addons
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=addons
baseurl=http://mirror.clarkson.edu/centos/$releasever/addons/$basearch/
gpgcheck=1
protect=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5

#additional packages that may be useful
[extras]
name=CentOS-$releasever - Extras
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras
baseurl=http://mirror.clarkson.edu/centos/$releasever/extras/$basearch/
gpgcheck=1
protect=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5

#additional packages that extend functionality of existing packages
[centosplus]
name=CentOS-$releasever - Plus
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=centosplus
baseurl=http://mirror.clarkson.edu/centos/$releasever/centosplus/$basearch/
gpgcheck=1
enabled=0
protect=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5

#contrib - packages by Centos Users
[contrib]
name=CentOS-$releasever - Contrib
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=contrib
baseurl=http://mirror.clarkson.edu/centos/$releasever/contrib/$basearch/
gpgcheck=1
enabled=0
protect=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5
    • Edited /etc/yum.repos.d/
# Name: RPMforge RPM Repository for Red Hat Enterprise 5 - dag
# URL: http://rpmforge.net/
[rpmforge]
name = Red Hat Enterprise $releasever - RPMforge.net - dag
baseurl = http://mirror.clarkson.edu/dag/redhat/el5/en/$basearch/dag
#mirrorlist = http://apt.sw.be/redhat/el5/en/mirrors-rpmforge
#mirrorlist = file:///etc/yum.repos.d/mirrors-rpmforge
enabled = 1
protect = 0
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rpmforge-dag
gpgcheck = 1
  • Installed Yum ProtectBase (Note: This must be installed prior to installing the packages below.)
    • yum install yum-protectbase
  • yum install yum-fastestmirror vim-enhanced gcc emacs-nox
  • yum update

Created User

  • Created user mccarrms
    • /usr/sbin/useradd -m mccarrms
  • Set password for mccarrms
    • passwd mccarrms

Configured Sudo

  • /usr/sbin/visudo
## Sudoers allows particular users to run various commands as
## the root user, without needing the root password.

#User_Alias ADMINS = mccarrms

## Networking
Cmnd_Alias NETWORKING = /sbin/route, /sbin/ifconfig, /bin/ping, /sbin/dhclient, /usr/bin/net, /sbin/iptables, /usr/bin/rfcomm, /usr/bin/wvdial, /sbin/iwconfig, /sbin/mii-tool

## Installation and management of software
Cmnd_Alias SOFTWARE = /bin/rpm, /usr/bin/up2date, /usr/bin/yum

## Services
Cmnd_Alias SERVICES = /sbin/service, /sbin/chkconfig

## Updating the locate database
Cmnd_Alias LOCATE = /usr/sbin/updatedb

## Storage
Cmnd_Alias STORAGE = /sbin/fdisk, /sbin/sfdisk, /sbin/parted, /sbin/partprobe, /bin/mount, /bin/umount

## Delegating permissions
Cmnd_Alias DELEGATING = /usr/sbin/visudo, /bin/chown, /bin/chmod, /bin/chgrp

## Processes
Cmnd_Alias PROCESSES = /bin/nice, /bin/kill, /usr/bin/kill, /usr/bin/killall

## Drivers
Cmnd_Alias DRIVERS = /sbin/modprobe

## Shells
Cmnd_Alias SHELLS = /bin/sh, /bin/bash, /usr/bin/rsh, /bin/dash, /bin/rbash, /bin/su

## Users
Cmnd_Alias USERS = /usr/sbin/useradd, /usr/sbin/userdel, /usr/sbin/userhelper, /usr/sbin/usermod, /usr/sbin/usernetctl

Defaults    requiretty

Defaults    env_reset,tty_tickets,lecture=always,logfile=/var/log/sudo.log
Defaults    env_keep = "COLORS DISPLAY HOSTNAME HISTSIZE INPUTRC KDEDIR \
                        LS_COLORS MAIL PS1 PS2 QTDIR USERNAME \
                        LANG LC_ADDRESS LC_CTYPE LC_COLLATE LC_IDENTIFICATION \
                        LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC \
                        LC_PAPER LC_TELEPHONE LC_TIME LC_ALL LANGUAGE LINGUAS \
                        _XKB_CHARSET XAUTHORITY"

## Allow root to run any commands anywhere
root    ALL=(ALL)       ALL
%wheel  ALL=(ALL)       ALL
%admins ALL=(root)      ALL, !SHELLS
%xenadmins ALL=/usr/sbin/xm

Configured Networks

  • Configured hostname in /etc/sysconfig/network
NETWORKING=yes
NETWORKING_IPV6=no
HOSTNAME=xen1.cslabs.clarkson.edu
GATEWAY=128.153.145.1
  • Verified eth0 configuration for Clarkson Network in /etc/sysconfig/network-scripts/ifcfg-eth0
# Intel Corporation 82541PI Gigabit Ethernet Controller
DEVICE=eth0
BOOTPROTO=static
DHCPCLASS=
HWADDR=00:1B:21:28:C8:56
IPADDR=128.153.145.41
NETMASK=255.255.255.0
ONBOOT=yes
  • Verified eth1 configuration for the Server Room Network in /etc/sysconfig/network-scripts/ifcfg-eth1
# Intel Corporation 80003ES2LAN Gigabit Ethernet Controller (Copper)
DEVICE=eth1
BOOTPROTO=static
DHCPCLASS=
HWADDR=00:E0:81:B5:88:74
IPADDR=10.0.1.37
NETMASK=255.255.255.0
ONBOOT=yes
  • Verified eth2 configuration for the Internal Network in /etc/sysconfig/network-scripts/ifcfg-eth2
# Intel Corporation 80003ES2LAN Gigabit Ethernet Controller (Copper)
DEVICE=eth2
BOOTPROTO=static
DHCPCLASS=
HWADDR=00:E0:81:B5:88:75
IPADDR=10.0.0.16
NETMASK=255.255.255.0
ONBOOT=yes

Configured Additional Xen Bridges

  • Created a network configuration file (/etc/xen/scripts/networks-cslabs)
#!/bin/sh
# Exit if anything goes wrong.
set -e
# First arg is the operation.
OP=$1
shift
script=/etc/xen/scripts/network-bridge-cslabs
case ${OP} in
start)
        $script start vifnum=2 bridge=xenbr2 netdev=eth2
        $script start vifnum=1 bridge=xenbr1 netdev=eth1
        $script start vifnum=0 bridge=xenbr0 netdev=eth0
        ;;
stop)
        $script stop vifnum=2 bridge=xenbr2 netdev=eth2
        $script stop vifnum=1 bridge=xenbr1 netdev=eth1
        $script stop vifnum=0 bridge=xenbr0 netdev=eth0
        ;;
status)
        $script status vifnum=2 bridge=xenbr2 netdev=eth2
        $script status vifnum=1 bridge=xenbr1 netdev=eth1
        $script status vifnum=0 bridge=xenbr0 netdev=eth0
        ;;
*)
        echo 'Unknown command: ' ${OP}
        echo 'Valid commands are: start, stop, status'
        exit 1
esac
  • Made script executable
    • chmod +x /etc/xen/scripts/networks-cslabs
  • Copied and modified /etc/xen/scripts/network-bridge script to prevent peth0: received packet with own address as source address errors due to MAC address conflicts between dom0s residing on the same network. (Fix created based on patch submitted by lab alum Cyrus Katrak - Original Bug Report
    • cp /etc/xen/scripts/network-bridge /etc/xen/scripts/network-bridge-cslabs
#!/bin/sh
#============================================================================
# Default Xen network start/stop script.
# Xend calls a network script when it starts.
# The script name to use is defined in /etc/xen/xend-config.sxp
# in the network-script field.
#
# This script creates a bridge (default xenbr${vifnum}), adds a device
# (default eth${vifnum}) to it, copies the IP addresses from the device
# to the bridge and adjusts the routes accordingly.
#
# If all goes well, this should ensure that networking stays up.
# However, some configurations are upset by this, especially
# NFS roots. If the bridged setup does not meet your needs,
# configure a different script, for example using routing instead.
#
# Usage:
#
# network-bridge (start|stop|status) {VAR=VAL}*
#
# Vars:
#
# vifnum     Virtual device number to use (default 0). Numbers >=8
#            require the netback driver to have nloopbacks set to a
#            higher value than its default of 8.
# bridge     The bridge to use (default xenbr${vifnum}).
# netdev     The interface to add to the bridge (default eth${vifnum}).
# antispoof  Whether to use iptables to prevent spoofing (default no).
#
# Internal Vars:
# pdev="p${netdev}"
# vdev="veth${vifnum}"
# vif0="vif0.${vifnum}"
#
# start:
# Creates the bridge
# Copies the IP and MAC addresses from netdev to vdev
# Renames netdev to be pdev 
# Renames vdev to be netdev 
# Enslaves pdev, vdev to bridge
#
# stop:
# Removes netdev from the bridge
# Transfers addresses, routes from netdev to pdev
# Renames netdev to vdev
# Renames pdev to netdev 
# Deletes bridge
#
# status:
# Print addresses, interfaces, routes
#
#============================================================================

#macid is used to uniquely identify this dom0 on this network
#change this to avoid MAC address conflicts if you get:
#"peth0: received packet with own address as source address"
macid="E0"

dir=$(dirname "$0")
. "$dir/xen-script-common.sh"
. "$dir/xen-network-common.sh"

findCommand "$@"
evalVariables "$@"

vifnum=${vifnum:-$(ip route list | awk '/^default / { print $NF }' | sed 's/^[^0-9]*//')}
vifnum=${vifnum:-0}
bridge=${bridge:-xenbr${vifnum}}
netdev=${netdev:-eth${vifnum}}
antispoof=${antispoof:-no}

pdev="p${netdev}"
vdev="veth${vifnum}"
vif0="vif0.${vifnum}"
addr_pfx=

get_ip_info() {
    addr_pfx=`ip addr show dev $1 | sed -n 's/^ *inet \(.*\) [^ ]*$/\1/p'`
    gateway=`ip route show dev $1 | fgrep default | sed 's/default via //'`
}
    
is_bonding() {
    [ -f "/sys/class/net/$1/bonding/slaves" ]
}

is_vlan() {
    [ -f "/proc/net/vlan/$1" ]
}

is_ifup() {
    ip link show dev $1 | awk '{ exit $3 !~ /[<,]UP[,>]/ }'
}

do_ifup() {
    if ! ifup $1 || ! is_ifup $1 ; then
        if [ -n "${addr_pfx}" ] ; then
            # use the info from get_ip_info()
            ip addr flush $1
            ip addr add ${addr_pfx} dev $1
            ip link set dev $1 up
            [ ${gateway} ] && ip route add default via ${gateway}
        fi
    fi
}

# Usage: transfer_addrs src dst
# Copy all IP addresses (including aliases) from device $src to device $dst.
transfer_addrs () {
    local src=$1
    local dst=$2
    # Don't bother if $dst already has IP addresses.
    if ip addr show dev ${dst} | egrep -q '^ *inet ' ; then
        return
    fi
    # Address lines start with 'inet' and have the device in them.
    # Replace 'inet' with 'ip addr add' and change the device name $src
    # to 'dev $src'.
    ip addr show dev ${src} | egrep '^ *inet ' | sed -e "
s/inet/ip addr add/
s@\([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+/[0-9]\+\)@\1@
s/${src}/dev ${dst} label ${dst}/
s/secondary//
" | sh -e
    # Remove automatic routes on destination device
    ip route list | sed -ne "
/dev ${dst}\( \|$\)/ {
  s/^/ip route del /
  p
}" | sh -e
}

# Usage: transfer_routes src dst
# Get all IP routes to device $src, delete them, and
# add the same routes to device $dst.
# The original routes have to be deleted, otherwise adding them
# for $dst fails (duplicate routes).
transfer_routes () {
    local src=$1
    local dst=$2
    # List all routes and grep the ones with $src in.
    # Stick 'ip route del' on the front to delete.
    # Change $src to $dst and use 'ip route add' to add.
    ip route list | sed -ne "
/dev ${src}\( \|$\)/ {
  h
  s/^/ip route del /
  P
  g
  s/${src}/${dst}/
  s/^/ip route add /
  P
  d
}" | sh -e
}


##
# link_exists interface
#
# Returns 0 if the interface named exists (whether up or down), 1 otherwise.
#
link_exists()
{
    if ip link show "$1" >/dev/null 2>/dev/null
    then
        return 0
    else
        return 1
    fi
}

# Set the default forwarding policy for $dev to drop.
# Allow forwarding to the bridge.
antispoofing () {
    iptables -P FORWARD DROP
    iptables -F FORWARD
    iptables -A FORWARD -m physdev --physdev-in ${pdev} -j ACCEPT
    iptables -A FORWARD -m physdev --physdev-in ${vif0} -j ACCEPT
}

# Usage: show_status dev bridge
# Print ifconfig and routes.
show_status () {
    local dev=$1
    local bridge=$2
    
    echo '============================================================'
    ip addr show ${dev}
    ip addr show ${bridge}
    echo ' '
    brctl show ${bridge}
    echo ' '
    ip route list
    echo ' '
    route -n
    echo '============================================================'
}

is_network_root () {
    local rootfs=$(awk '{ if ($1 !~ /^[ \t]*#/ && $2 == "/") { print $3; }}' /etc/mtab)
    local rootopts=$(awk '{ if ($1 !~ /^[ \t]*#/ && $2 == "/") { print $4; }}' /etc/mtab)

    [[ "$rootfs" =~ "^nfs" ]] || [[ "$rootopts" =~ "_netdev" ]] && return 0 || return 1
}

op_start () {
    if [ "${bridge}" = "null" ] ; then
	return
    fi

    if is_network_root ; then
        [ -x /usr/bin/logger ] && /usr/bin/logger "network-bridge: bridging not supported on network root; not starting"
        return
    fi

    if ! link_exists "$vdev"; then
        if link_exists "$pdev"; then
            # The device is already up.
            return
        else
            echo "
Link $vdev is missing.
This may be because you have reached the limit of the number of interfaces
that the loopback driver supports.  If the loopback driver is a module, you
may raise this limit by passing it as a parameter (nloopbacks=<N>); if the
driver is compiled statically into the kernel, then you may set the parameter
using loopback.nloopbacks=<N> on the domain 0 kernel command line.
" >&2
            exit 1
        fi
    fi

    create_bridge ${bridge}

    if link_exists "$vdev"; then
	mac=`ip link show ${netdev} | grep 'link\/ether' | sed -e 's/.*ether \(..:..:..:..:..:..\).*/\1/'`
	preiftransfer ${netdev}
	transfer_addrs ${netdev} ${vdev}
	if is_bonding ${netdev} || is_vlan ${netdev} || ! ifdown ${netdev}; then
	    # Remember the IP details if necessary.
	    get_ip_info ${netdev}
	    ip link set ${netdev} down
	    ip addr flush ${netdev}
	fi
	ip link set ${netdev} name ${pdev}
	ip link set ${vdev} name ${netdev}

	setup_bridge_port ${pdev}
	setup_bridge_port ${vif0}
	ip link set ${netdev} addr ${mac} arp on

 	ip link set ${pdev} addr fe:ff:ff:ff:${macid}:0${vifnum}
 	ip link set ${vif0} addr fe:ff:ff:ff:${macid}:0${vifnum}

	ip link set ${bridge} up
	add_to_bridge  ${bridge} ${vif0}
	add_to_bridge2 ${bridge} ${pdev}
	do_ifup ${netdev}
    else
	# old style without ${vdev}
	transfer_addrs  ${netdev} ${bridge}
	transfer_routes ${netdev} ${bridge}
    fi

    if [ ${antispoof} = 'yes' ] ; then
	antispoofing
    fi
}

op_stop () {
    if [ "${bridge}" = "null" ]; then
	return
    fi
    if ! link_exists "$bridge"; then
	return
    fi

    if link_exists "$pdev"; then
	ip link set dev ${vif0} down
	mac=`ip link show ${netdev} | grep 'link\/ether' | sed -e 's/.*ether \(..:..:..:..:..:..\).*/\1/'`
	transfer_addrs ${netdev} ${pdev}
	if ! ifdown ${netdev}; then
	    get_ip_info ${netdev}
	fi
	ip link set ${netdev} down arp off
	ip link set ${netdev} addr fe:ff:ff:ff:ff:ff
	ip link set ${pdev} down
	ip addr flush ${netdev}
	ip link set ${pdev} addr ${mac} arp on

	brctl delif ${bridge} ${pdev}
	brctl delif ${bridge} ${vif0}
	ip link set ${bridge} down

	ip link set ${netdev} name ${vdev}
	ip link set ${pdev} name ${netdev}
	do_ifup ${netdev}
    else
	transfer_routes ${bridge} ${netdev}
	ip link set ${bridge} down
    fi
    brctl delbr ${bridge}
}

# adds $dev to $bridge but waits for $dev to be in running state first
add_to_bridge2() {
    local bridge=$1
    local dev=$2
    local maxtries=10

    echo -n "Waiting for ${dev} to negotiate link."
    ip link set ${dev} up
    for i in `seq ${maxtries}` ; do
	if ifconfig ${dev} | grep -q RUNNING ; then
	    break
	else
	    echo -n '.'
	    sleep 1
	fi
    done

    if [ ${i} -eq ${maxtries} ] ; then echo '(link isnt in running state)' ; fi

    add_to_bridge ${bridge} ${dev}
}

case "$command" in
    start)
	op_start
	;;
    
    stop)
	op_stop
	;;

    status)
	show_status ${netdev} ${bridge}
	;;

    *)
	echo "Unknown command: $command" >&2
	echo 'Valid commands are: start, stop, status' >&2
	exit 1
esac
  • Edited /etc/xen/xend-config.sxp
(xend-unix-server yes)

(xend-unix-path /var/lib/xend/xend-socket)

(xend-relocation-hosts-allow '^localhost$ ^localhost\\.localdomain$')

(network-script networks-cslabs)

(vif-script vif-bridge)

(dom0-min-mem 256)

(dom0-cpus 0)

(vncpasswd '')

Configured Hosts

  • Edited /etc/hosts
127.0.0.1       localhost.localdomain localhost
::1             localhost6.localdomain6 localhost6
128.153.145.41  xen1.cslabs.clarkson.edu xen1.cslabs xen1
10.0.1.37       xen1.sr.cslabs.clarkson.edu xen1.sr.cslabs xen1.sr
10.0.0.16       xen1.int.cslabs.clarkson.edu xen1.int.cslabs xen1.int

Configured DNS Servers

  • Edited /etc/resolv.conf
search clarkson.edu
nameserver 128.153.0.254
nameserver 128.153.5.254

Configured IPtables

Due to the sensitivity of this material, this config file has been left off; however, the following rules are needed.
  • Restarted iptables
    • /etc/init.d/iptables restart

Configured SSH

  • Edited /etc/ssh/sshd_config
Due to the sensitivity of this material, this config file has been left off.
  • Restarted sshd
    • /etc/init.d/sshd restart

Set Up SSH Login Banner

  • Edited /etc/issue.net
               ___
 __ _____ ___ <  /
 \ \ / -_) _ \/ / 
/_\_\\__/_//_/_/  
                    

Configured Password Requirements

  • Edited /etc/login.defs
MAIL_DIR        /var/spool/mail

PASS_MAX_DAYS   360
PASS_MIN_DAYS   0
PASS_MIN_LEN    8
PASS_WARN_AGE   60

UID_MIN                   500
UID_MAX                 60000

GID_MIN                   500
GID_MAX                 60000

CREATE_HOME     yes

UMASK           077

USERGROUPS_ENAB yes

MD5_CRYPT_ENAB yes

ENCRYPT_METHOD MD5

Added Custom PATH Variables

  • Added the following to /etc/profile
PATH=$PATH:/usr/sbin:/sbin
export PATH

Modified Root's Crontab

  • crontab -e
# Used to update locate database
0 * * * * /usr/bin/updatedb

Set Up & Configured NTP

  • Installed NTP
    • yum install ntp
  • Edited /etc/ntp.conf
restrict default kod nomodify notrap nopeer noquery
restrict -6 default kod nomodify notrap nopeer noquery

restrict 127.0.0.1
restrict -6 ::1

restrict tick.clarkson.edu mask 255.255.255.255 nomodify notrap noquery
restrict tock.clarkson.edu mask 255.255.255.255 nomodify notrap noquery

server tick.clarkson.edu
server tock.clarkson.edu

server  127.127.1.0     # local clock
fudge   127.127.1.0 stratum 10

driftfile /var/lib/ntp/drift

keys /etc/ntp/keys
  • Edited /etc/ntp/step-tickers
tick.clarkson.edu
tock.clarkson.edu
  • Configured ntpd to start on boot
    • /sbin/chkconfig --levels 2345 ntpd on
  • Started ntpd
    • /etc/init.d/ntpd start

Configured ntpd to Sync Hardware Clock

  • Edited /etc/sysconfig/ntpd
# Drop root to id 'ntp:ntp' by default.
OPTIONS="-u ntp:ntp -p /var/run/ntpd.pid"

# Set to 'yes' to sync hw clock after successful ntpdate
SYNC_HWCLOCK=yes

# Additional options for ntpdate
NTPDATE_OPTIONS=""

Installed and Configured APCUPSD

This package is used to monitor the UPS which Xen1 is plugged into and is used to shutdown the system in the event of a power failure.

Configured to Power On when Power is Restored

  • Edited the BIOS to have Restore on AC/Power Loss set to Power On.

Installed and configured apcupsd

  • Installed apcupsd
    • yum install apcupsd
  • Edited /etc/apcupsd/apcupsd.conf
## apcupsd.conf v1.1 ##

UPSNAME ups4

UPSCABLE ether

UPSTYPE net
DEVICE 128.153.145.42:3551

LOCKFILE /var/lock

SCRIPTDIR /etc/apcupsd

PWRFAILDIR /etc/apcupsd

NOLOGINDIR /etc

ONBATTERYDELAY 6

BATTERYLEVEL 10

MINUTES 15

TIMEOUT 0

ANNOY 300

ANNOYDELAY 60

NOLOGON disable

KILLDELAY 0

NETSERVER on

NISIP 127.0.0.1

NISPORT 3551

EVENTSFILE /var/log/apcupsd.events

EVENTSFILEMAX 10

UPSCLASS standalone

UPSMODE disable

STATTIME 0

STATFILE /var/log/apcupsd.status

LOGSTATS off

DATATIME 0

SELFTEST 336
  • Configured apcupsd to start on boot
    • /sbin/chkconfig --levels 2345 apcupsd on
  • Started apcupsd
    • /etc/init.d/apcupsd start

Configured Xen Domains to Shutdown on System Shutdown

  • Edited the following lines in /etc/sysconfig/xendomains
XENDOMAINS_SAVE=""

XENDOMAINS_RESTORE=false

Installed Debootstrap

Configured Aliases

  • Edited /etc/aliases
#
#  Aliases in this file will NOT be expanded in the header from
#  Mail, but WILL be visible over networks or from /bin/mail.
#
#       >>>>>>>>>>      The program "newaliases" must be run after
#       >> NOTE >>      this file is updated for any changes to
#       >>>>>>>>>>      show through to sendmail.
#

# Basic system aliases -- these MUST be present.
mailer-daemon:  postmaster
postmaster:     logwatch@cslabs.clarkson.edu

# General redirections for pseudo accounts.
bin:            logwatch@cslabs.clarkson.edu
daemon:         logwatch@cslabs.clarkson.edu
adm:            logwatch@cslabs.clarkson.edu
lp:             logwatch@cslabs.clarkson.edu
sync:           logwatch@cslabs.clarkson.edu
shutdown:       logwatch@cslabs.clarkson.edu
halt:           logwatch@cslabs.clarkson.edu
mail:           logwatch@cslabs.clarkson.edu
news:           logwatch@cslabs.clarkson.edu
uucp:           logwatch@cslabs.clarkson.edu
operator:       logwatch@cslabs.clarkson.edu
games:          logwatch@cslabs.clarkson.edu
gopher:         logwatch@cslabs.clarkson.edu
ftp:            logwatch@cslabs.clarkson.edu
nobody:         logwatch@cslabs.clarkson.edu
radiusd:        logwatch@cslabs.clarkson.edu
nut:            logwatch@cslabs.clarkson.edu
dbus:           logwatch@cslabs.clarkson.edu
vcsa:           logwatch@cslabs.clarkson.edu
canna:          logwatch@cslabs.clarkson.edu
wnn:            logwatch@cslabs.clarkson.edu
rpm:            logwatch@cslabs.clarkson.edu
nscd:           logwatch@cslabs.clarkson.edu
pcap:           logwatch@cslabs.clarkson.edu
apache:         logwatch@cslabs.clarkson.edu
webalizer:      logwatch@cslabs.clarkson.edu
dovecot:        logwatch@cslabs.clarkson.edu
fax:            logwatch@cslabs.clarkson.edu
quagga:         logwatch@cslabs.clarkson.edu
radvd:          logwatch@cslabs.clarkson.edu
pvm:            logwatch@cslabs.clarkson.edu
amanda:         logwatch@cslabs.clarkson.edu
privoxy:        logwatch@cslabs.clarkson.edu
ident:          logwatch@cslabs.clarkson.edu
named:          logwatch@cslabs.clarkson.edu
xfs:            logwatch@cslabs.clarkson.edu
gdm:            logwatch@cslabs.clarkson.edu
mailnull:       logwatch@cslabs.clarkson.edu
postgres:       logwatch@cslabs.clarkson.edu
sshd:           logwatch@cslabs.clarkson.edu
smmsp:          logwatch@cslabs.clarkson.edu
postfix:        logwatch@cslabs.clarkson.edu
netdump:        logwatch@cslabs.clarkson.edu
ldap:           logwatch@cslabs.clarkson.edu
squid:          logwatch@cslabs.clarkson.edu
ntp:            logwatch@cslabs.clarkson.edu
mysql:          logwatch@cslabs.clarkson.edu
desktop:        logwatch@cslabs.clarkson.edu
rpcuser:        logwatch@cslabs.clarkson.edu
rpc:            logwatch@cslabs.clarkson.edu
nfsnobody:      logwatch@cslabs.clarkson.edu

ingres:         logwatch@cslabs.clarkson.edu
system:         logwatch@cslabs.clarkson.edu
toor:           logwatch@cslabs.clarkson.edu
manager:        logwatch@cslabs.clarkson.edu
dumper:         logwatch@cslabs.clarkson.edu
abuse:          logwatch@cslabs.clarkson.edu

newsadm:        news
newsadmin:      news
usenet:         news
ftpadm:         ftp
ftpadmin:       ftp
ftp-adm:        ftp
ftp-admin:      ftp
www:            webmaster
webmaster:      logwatch@cslabs.clarkson.edu
noc:            logwatch@cslabs.clarkson.edu
security:       logwatch@cslabs.clarkson.edu
hostmaster:     logwatch@cslabs.clarkson.edu
info:           postmaster
marketing:      postmaster
sales:          postmaster
support:        postmaster


# trap decode to catch security attacks
decode:         logwatch@cslabs.clarkson.edu

# Person who should get roots's mail
root:           logwatch@cslabs.clarkson.edu
  • Updated aliases
    • /usr/bin/newaliases