Difference between revisions of "Mirror Setup Process"

From CSLabsWiki
Jump to: navigation, search
(New page: This page summarizes how Mirror was set up in Spring 2009. ==Install== *Installed CentOS 5.3 x64. **Partition Scheme *** *** *** ==Configuration== ===Updated VM=== *Added RPMForge Yu...)
 
 
(78 intermediate revisions by 3 users not shown)
Line 1: Line 1:
This page summarizes how [[Mirror]] was set up in Spring 2009.
+
[[Category:Server Setup Documentation]]
  
==Install==
+
This page summarizes how [[Old Mirror]] was set up in Spring 2014.
*Installed CentOS 5.3 x64.
 
**Partition Scheme
 
***
 
***
 
***
 
  
==Configuration==
+
=Install=
===Updated VM===
+
See [[Debian Server Setup]]
*Added RPMForge Yum Repository
 
**<code>rpm -Uhv http://apt.sw.be/redhat/el5/en/x86_64/rpmforge/RPMS//rpmforge-release-0.3.6-1.el5.rf.x86_64.rpm</code>
 
***From [http://dag.wieers.com/rpm/FAQ.php#B2 Dag Wieers]
 
  
*<code>yum install yum-fastestmirror vim-enhanced gcc emacs-nox screen</code>
+
==Debian 7 x64==
*<code>yum update</code>
+
*71 GB  / - Software RAID 1
 +
*4GB Swap per OS Drive
  
===Created Users===
+
=Setup=
*Created user mccarrms
+
See [[Debian Server Setup]]
**<code>/usr/sbin/useradd -m mccarrms</code>
 
*Set password for mccarrms
 
**<code>passwd mccarrms</code>
 
*Created user ignazirj
 
**<code>/usr/sbin/useradd -m ignazirj</code>
 
*Set password for ignazirj
 
**<code>passwd ignazirj</code>
 
  
 +
==Raid==
 +
*sda sdb sdc sde sdg sdh
 +
*/dev/md2
 +
*Software RAID 10
 +
*/storage xfs
  
===Configured Sudo===
+
==Networking==
*<code>/usr/sbin/visudo</code>
 
  
<code><pre>
+
===eth2===
## Sudoers allows particular users to run various commands as
+
*Intel Corporation 82541PI Gigabit Ethernet Controller
## the root user, without needing the root password.
+
*HWADDR=00:1B:21:28:C8:48
 +
*IPADDR=128.153.145.19
  
## Networking
+
===eth1===
Cmnd_Alias NETWORKING = /sbin/route, /sbin/ifconfig, /bin/ping, /sbin/dhclient, /usr/bin/net, /sbin/iptables, /usr/bin/rfcomm, /usr/bin/wvdial, /sbin/iwconfig, /sbin/mii-tool
+
*Intel Corporation 82566DM-2 Gigabit Network Connection
 +
*HWADDR=00:30:48:9A:DB:26
 +
*IPADDR=10.0.1.36
  
## Installation and management of software
+
===eth3===
Cmnd_Alias SOFTWARE = /bin/rpm, /usr/bin/up2date, /usr/bin/yum
+
*Intel Corporation 82573L Gigabit Ethernet Controller
 +
*HWADDR=00:30:48:9A:DB:27
 +
*IPADDR=10.0.0.14
  
## Services
+
==Set Up SSH Login Banner==
Cmnd_Alias SERVICES = /sbin/service, /sbin/chkconfig
+
/etc/issue.net
 
+
<code><pre>
## Updating the locate database
+
        _               
Cmnd_Alias LOCATE = /usr/sbin/updatedb
+
  __ _  (_)__________  ____
 
+
/ ' \/ / __/ __/ _ \/ __/
## Storage
+
/_/_/_/_/_/ /_/ \___/_/  
Cmnd_Alias STORAGE = /sbin/fdisk, /sbin/sfdisk, /sbin/parted, /sbin/partprobe, /bin/mount, /bin/umount
+
                         
 
+
</pre></code>
## Delegating permissions
+
==SSHD config==
Cmnd_Alias DELEGATING = /usr/sbin/visudo, /bin/chown, /bin/chmod, /bin/chgrp
+
uncomment
 
+
Banner /etc/issue.net
## Processes
 
Cmnd_Alias PROCESSES = /bin/nice, /bin/kill, /usr/bin/kill, /usr/bin/killall
 
 
 
## Drivers
 
Cmnd_Alias DRIVERS = /sbin/modprobe
 
 
 
## Shells
 
Cmnd_Alias SHELLS = /bin/sh, /bin/bash, /usr/bin/rsh, /bin/dash, /bin/rbash, /bin/su
 
 
 
## Users
 
Cmnd_Alias USERS = /usr/sbin/useradd, /usr/sbin/userdel, /usr/sbin/userhelper, /usr/sbin/usermod, /usr/sbin/usernetctl
 
 
 
Defaults    requiretty
 
  
Defaults    env_reset,tty_tickets,lecture=always,logfile=/var/log/sudo.log
+
==Installed nginx==
Defaults    env_keep = "COLORS DISPLAY HOSTNAME HISTSIZE INPUTRC KDEDIR \
+
apt-get install install nginx
                        LS_COLORS MAIL PS1 PS2 QTDIR USERNAME \
 
                        LANG LC_ADDRESS LC_CTYPE LC_COLLATE LC_IDENTIFICATION \
 
                        LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC \
 
                        LC_PAPER LC_TELEPHONE LC_TIME LC_ALL LANGUAGE LINGUAS \
 
                        _XKB_CHARSET XAUTHORITY"
 
  
## Allow root to run any commands anywhere
+
==Configure rsync to run as a daemon==
root    ALL=(ALL)      ALL
+
*Installed <code>xinetd</code>
%wheel  ALL=(ALL)      ALL
+
apt-get install rsync
</pre></code>
 
  
===Configured Networks===
+
*Configured rsync /etc/rsyncd.conf
*Configured hostname in <code>/etc/sysconfig/network</code>
 
 
<code><pre>
 
<code><pre>
NETWORKING=yes
+
uid = nobody
NETWORKING_IPV6=no
+
gid = nogroup
HOSTNAME=mirror.clarkson.edu
+
use chroot = yes
GATEWAY=128.153.145.1
+
max connections = 20
</pre></code>
+
pid file = /var/run/rsyncd.pid
 +
motd file = /etc/rsyncd.motd
 +
log file = /var/log/rsync.log
 +
transfer logging = yes
 +
log format = %t %a %m %f %b
 +
syslog facility = local3
 +
timeout = 900
 +
dont compress = *.gz *.tgz *.zip *.z *.Z *.rpm *.deb *.bz2
 +
refuse options = checksum
  
*Verified eth0 configuration for Clarkson Network in <code>/etc/sysconfig/network-scripts/ifcfg-eth0</code>
+
[name]
<code><pre>
+
  comment = Name of Repository
# Intel Corporation 82541PI Gigabit Ethernet Controller
+
  path = /storage/repository
DEVICE=eth0
+
  exclude = lost+found/
BOOTPROTO=static
+
  read only = true
BROADCAST=128.153.145.255
+
  ignore nonreadable = yes</pre></code>
HWADDR=00:1B:21:28:C8:48
 
IPADDR=128.153.145.19
 
NETMASK=255.255.255.0
 
NETWORK=128.153.145.0
 
ONBOOT=yes
 
</pre></code>
 
  
*Verified eth1 configuration for the Server Room Network in <code>/etc/sysconfig/network-scripts/ifcfg-eth1</code>
+
*Created rsync motd /etc/rsyncd.motd
 
<code><pre>
 
<code><pre>
# Intel Corporation 82566DM-2 Gigabit Network Connection
+
###############################################################
DEVICE=eth1
 
BOOTPROTO=static
 
BROADCAST=10.0.1.255
 
HWADDR=00:30:48:9A:DB:26
 
IPADDR=10.0.1.36
 
NETMASK=255.255.255.0
 
NETWORK=10.0.1.0
 
ONBOOT=yes
 
</pre></code>
 
  
*Verified eth2 configuration for the Internal Network in <code>/etc/sysconfig/network-scripts/ifcfg-eth2</code>
+
The Clarkson University mirror is located in Potsdam, NY, US.
<code><pre>
 
# Intel Corporation 82573L Gigabit Ethernet Controller
 
DEVICE=eth2
 
BOOTPROTO=static
 
BROADCAST=10.0.0.255
 
HWADDR=00:30:48:9A:DB:27
 
IPADDR=10.0.0.14
 
NETMASK=255.255.255.0
 
NETWORK=10.0.0.0
 
ONBOOT=yes
 
</pre></code>
 
  
====Configured Hosts====
+
This mirror is operated by the Clarkson Open Source Institute.
*Edited <code>/etc/hosts</code>
+
http://cosi.clarkson.edu/
<code><pre>
 
127.0.0.1      localhost.localdomain localhost
 
::1            localhost6.localdomain6 localhost6
 
128.153.145.19  mirror.clarkson.edu mirror.cslabs.clarkson.edu mirror.cslabs mirror
 
10.0.1.36      mirror.sr.cslabs.clarkson.edu mirror.sr.cslabs mirror.sr
 
10.0.0.14      mirror.int.cslabs.clarkson.edu mirror.int.cslabs mirror.int
 
</pre></code>
 
  
====Configured DNS Servers====
+
If you have any questions or problems, please email
*Edited <code>/etc/resolv.conf</code>
+
mirror-admin@cslabs.clarkson.edu.
<code><pre>
 
search clarkson.edu
 
nameserver 128.153.0.254
 
nameserver 128.153.5.254
 
</pre></code>
 
  
===Configured IPtables===
+
Recent changes can be found at
<code><pre>
+
http://status.cslabs.clarkson.edu/tag/mirror.
Due to the sensitivity of this material, this config file has been left off; however, the following rules are needed.
 
</pre></code>
 
*Restarted iptables
 
**<code>/etc/init.d/iptables restart</code>
 
  
===Configured SSH===
+
###############################################################
*Edited <code>/etc/ssh/sshd_config</code>
 
<code><pre>
 
Due to the sensitivity of this material, this config file has been left off.
 
 
</pre></code>
 
</pre></code>
*Restarted sshd
 
**<code>/etc/init.d/sshd restart</code>
 
  
====Set Up SSH Login Banner====
+
==Set Up Fedora Mirror Manager==
*Edited <code>/etc/issue.net</code>
+
*Installed mirrormanager
<code><pre>
+
cd /usr/local/sbin
        _               
+
  git clone git://git.fedorahosted.org/mirrormanager/
  __ _  (_)__________  ____
+
ln -s /usr/local/sbin/mirrormanager/client/report_mirror /usr/local/sbin/report_mirror
  / ' \/ / __/ __/ _ \/ __/
+
/_/_/_/_/_/ /_/ \___/_/  
 
                         
 
</pre></code>
 
  
===Configured Password Requirements===
+
*Configured mirrormanager
*Edited <code>/etc/login.defs</code>
+
**Modified /etc/mirrormanager-client/report_mirror.conf
 
<code><pre>
 
<code><pre>
MAIL_DIR        /var/spool/mail
+
[global]
 +
# if enabled=0, no data is sent to the database
 +
enabled=1
 +
# server= is the URL to the MirrorManager XML-RPC interface
 +
server=https://admin.fedoraproject.org/mirrormanager/xmlrpc
  
PASS_MAX_DAYS  360
 
PASS_MIN_DAYS  0
 
PASS_MIN_LEN    8
 
PASS_WARN_AGE  60
 
  
UID_MIN                  500
+
[site]
UID_MAX                60000
+
# if enabled=0, no data about this site is sent to the database
 +
enabled=1
 +
# Name and Password fields need to match the Site name and password
 +
# fields you entered for your Site in the MirrorManager database at
 +
# https://admin.fedoraproject.org/mirrormanager
 +
name=Clarkson University
 +
password=<PASSWORD GOES HERE>
  
GID_MIN                  500
+
[host]
GID_MAX                60000
+
# if enabled=0, no data about this host is sent to the database
 +
enabled=1
 +
# Name field need to match the Host name field you entered for your
 +
# Host in the MirrorManager database at
 +
# https://admin.fedoraproject.org/mirrormanager
 +
name=mirror.clarkson.edu
 +
# if user_active=0, no data about this category is given to the public
 +
# This can be used to toggle between serving and not serving data,
 +
# such enabled during the nighttime (when you have more idle bandwidth
 +
# available) and disabled during the daytime.
 +
# By not specifying user_active, the database will not be updated.
 +
# user_active=1
  
CREATE_HOME    yes
+
[stats]
 +
# Stats are only sent when run with the -s option
 +
# and when this section is enabled.
 +
# This feature is not presently implemented
 +
enabled=0
 +
apache=/var/log/httpd/access_log
 +
vsftpd=/var/log/vsftpd.log
 +
# remember to enable log file and transfer logging in rsyncd.conf
 +
rsyncd=/var/log/rsyncd.log
  
UMASK          077
 
  
USERGROUPS_ENAB yes
+
# Content Categories
 +
# These sections match the Categories for content tracked by MirrorManager.
 +
#
 +
# enabled=1 means information about this category will be sent to the database.
 +
# enabled=0, no data about this host is sent to the database.  If the
 +
# database already has information for you for this Category, it will
 +
# remain unchanged.  This can be used to update the database after you
 +
# have manually synced some infrequently-updated content, such as
 +
# historical releases.
 +
#
 +
# path= is the path on your local disk to the top-level directory for this Category
  
MD5_CRYPT_ENAB yes
+
[Fedora Linux]
 
+
enabled=1
ENCRYPT_METHOD MD5
+
path=/usr/share/nginx/www/fedora/linux
</pre></code>
 
  
===Added Custom PATH Variables===
+
[Fedora EPEL]
*Added the following to <code>/etc/profile</code>
+
enabled=1
<code><pre>
+
path=/usr/share/nginx/www/epel
PATH=$PATH:/usr/sbin/:/sbin/
 
export PATH
 
</pre></code>
 
  
===Set Up & Configured NTP===
+
# lesser used categories below
*Installed NTP
 
**<code>yum install ntp</code>
 
  
*Edited <code>/etc/ntp.conf</code>
+
[Fedora Web]
<code><pre>
+
enabled=0
restrict default kod nomodify notrap nopeer noquery
+
path=/usr/share/nginx/www/pub/fedora/web
restrict -6 default kod nomodify notrap nopeer noquery
 
  
restrict 127.0.0.1
+
[Fedora Secondary Arches]
restrict -6 ::1
+
enabled=0
 +
path=/usr/share/nginx/www/pub/fedora-secondary
  
restrict tick.clarkson.edu mask 255.255.255.255 nomodify notrap noquery
+
[Fedora Other]
restrict tock.clarkson.edu mask 255.255.255.255 nomodify notrap noquery
+
enabled=0
 +
path=/usr/share/nginx/www/pub/alt
  
server tick.clarkson.edu
+
# historical content
server tock.clarkson.edu
 
  
server  127.127.1.0     # local clock
+
[Fedora Core]
fudge  127.127.1.0 stratum 10
+
enabled=0
 +
path=/usr/share/nginx/www/pub/fedora/linux/core
  
driftfile /var/lib/ntp/drift
+
[Fedora Extras]
 +
enabled=0
 +
path=/usr/share/nginx/www/pub/fedora/linux/extras
  
keys /etc/ntp/keys
+
[Fedora Archive]
 +
enabled=0
 +
path=/usr/share/nginx/www/pub/fedora-archive
 
</pre></code>
 
</pre></code>
  
*Edited <code>/etc/ntp/step-tickers</code>
+
*Added cron entries for root
<code><pre>
 
tick.clarkson.edu
 
tock.clarkson.edu
 
</pre></code>
 
 
 
*Configured ntpd to start on boot
 
**<code>/sbin/chkconfig --levels 35 ntpd on</code>
 
 
 
*Started ntpd
 
**<code>/etc/init.d/ntpd start</code>
 
 
 
====Configured ntpd to Sync Hardware Clock====
 
*Edited <code>/etc/sysconfig/ntpd</code>
 
 
<code><pre>
 
<code><pre>
# Drop root to id 'ntp:ntp' by default.
+
# Dir sizes
OPTIONS="-u ntp:ntp -p /var/run/ntpd.pid"
+
0 0 * * * /usr/local/bin/dir_sizes.sh
  
# Set to 'yes' to sync hw clock after successful ntpdate
+
# backup
SYNC_HWCLOCK=yes
+
45 0 * * * /bin/nice -n 19 /usr/bin/ionice -c2 -n7 /usr/local/bin/backup.sh
  
# Additional options for ntpdate
+
10 0 * * * /usr/local/sbin/report_mirror
NTPDATE_OPTIONS=""
 
 
</pre></code>
 
</pre></code>
  
===Configured to Power On when Power is Restored===
 
*Edited the BIOS to have <code>Restore on AC/Power Loss</code> set to <code>Power On</code>.
 
  
[[Category:Documentation]]
+
[[mirror rsync setup|Setup Rsync Scripts]]
[[Category:Infrastructure]]
 

Latest revision as of 22:05, 27 April 2016


This page summarizes how Old Mirror was set up in Spring 2014.

Install

See Debian Server Setup

Debian 7 x64

  • 71 GB / - Software RAID 1
  • 4GB Swap per OS Drive

Setup

See Debian Server Setup

Raid

  • sda sdb sdc sde sdg sdh
  • /dev/md2
  • Software RAID 10
  • /storage xfs

Networking

eth2

  • Intel Corporation 82541PI Gigabit Ethernet Controller
  • HWADDR=00:1B:21:28:C8:48
  • IPADDR=128.153.145.19

eth1

  • Intel Corporation 82566DM-2 Gigabit Network Connection
  • HWADDR=00:30:48:9A:DB:26
  • IPADDR=10.0.1.36

eth3

  • Intel Corporation 82573L Gigabit Ethernet Controller
  • HWADDR=00:30:48:9A:DB:27
  • IPADDR=10.0.0.14

Set Up SSH Login Banner

/etc/issue.net

         _                 
  __ _  (_)__________  ____
 /  ' \/ / __/ __/ _ \/ __/
/_/_/_/_/_/ /_/  \___/_/   
                           

SSHD config

uncomment

Banner /etc/issue.net

Installed nginx

apt-get install install nginx

Configure rsync to run as a daemon

  • Installed xinetd
apt-get install rsync
  • Configured rsync /etc/rsyncd.conf
uid = nobody
gid = nogroup
use chroot = yes
max connections = 20
pid file = /var/run/rsyncd.pid
motd file = /etc/rsyncd.motd
log file = /var/log/rsync.log
transfer logging = yes
log format = %t %a %m %f %b
syslog facility = local3
timeout = 900
dont compress = *.gz *.tgz *.zip *.z *.Z *.rpm *.deb *.bz2
refuse options = checksum

[name]
  comment = Name of Repository
  path = /storage/repository
  exclude = lost+found/
  read only = true
  ignore nonreadable = yes
  • Created rsync motd /etc/rsyncd.motd
###############################################################

The Clarkson University mirror is located in Potsdam, NY, US.

This mirror is operated by the Clarkson Open Source Institute.
http://cosi.clarkson.edu/

If you have any questions or problems, please email
mirror-admin@cslabs.clarkson.edu.

Recent changes can be found at
http://status.cslabs.clarkson.edu/tag/mirror.

###############################################################

Set Up Fedora Mirror Manager

  • Installed mirrormanager
cd /usr/local/sbin
git clone git://git.fedorahosted.org/mirrormanager/
ln -s /usr/local/sbin/mirrormanager/client/report_mirror /usr/local/sbin/report_mirror

  • Configured mirrormanager
    • Modified /etc/mirrormanager-client/report_mirror.conf
[global]
# if enabled=0, no data is sent to the database
enabled=1
# server= is the URL to the MirrorManager XML-RPC interface
server=https://admin.fedoraproject.org/mirrormanager/xmlrpc


[site]
# if enabled=0, no data about this site is sent to the database
enabled=1
# Name and Password fields need to match the Site name and password
# fields you entered for your Site in the MirrorManager database at
# https://admin.fedoraproject.org/mirrormanager
name=Clarkson University
password=<PASSWORD GOES HERE>

[host]
# if enabled=0, no data about this host is sent to the database
enabled=1
# Name field need to match the Host name field you entered for your
# Host in the MirrorManager database at
# https://admin.fedoraproject.org/mirrormanager
name=mirror.clarkson.edu
# if user_active=0, no data about this category is given to the public
# This can be used to toggle between serving and not serving data,
# such enabled during the nighttime (when you have more idle bandwidth
# available) and disabled during the daytime.
# By not specifying user_active, the database will not be updated.
# user_active=1

[stats]
# Stats are only sent when run with the -s option
# and when this section is enabled.
# This feature is not presently implemented
enabled=0
apache=/var/log/httpd/access_log
vsftpd=/var/log/vsftpd.log
# remember to enable log file and transfer logging in rsyncd.conf
rsyncd=/var/log/rsyncd.log


# Content Categories
# These sections match the Categories for content tracked by MirrorManager.
#
# enabled=1 means information about this category will be sent to the database.
# enabled=0, no data about this host is sent to the database.  If the
# database already has information for you for this Category, it will
# remain unchanged.  This can be used to update the database after you
# have manually synced some infrequently-updated content, such as
# historical releases.
#
# path= is the path on your local disk to the top-level directory for this Category

[Fedora Linux]
enabled=1
path=/usr/share/nginx/www/fedora/linux

[Fedora EPEL]
enabled=1
path=/usr/share/nginx/www/epel

# lesser used categories below

[Fedora Web]
enabled=0
path=/usr/share/nginx/www/pub/fedora/web

[Fedora Secondary Arches]
enabled=0
path=/usr/share/nginx/www/pub/fedora-secondary

[Fedora Other]
enabled=0
path=/usr/share/nginx/www/pub/alt

# historical content

[Fedora Core]
enabled=0
path=/usr/share/nginx/www/pub/fedora/linux/core

[Fedora Extras]
enabled=0
path=/usr/share/nginx/www/pub/fedora/linux/extras

[Fedora Archive]
enabled=0
path=/usr/share/nginx/www/pub/fedora-archive
  • Added cron entries for root
# Dir sizes
0 0 * * * /usr/local/bin/dir_sizes.sh

# backup
45 0 * * * /bin/nice -n 19 /usr/bin/ionice -c2 -n7 /usr/local/bin/backup.sh

10 0 * * * /usr/local/sbin/report_mirror


Setup Rsync Scripts