2012年8月29日 星期三

handout for hypertable cluster installation and deployment

Base on hadoop/hdfs installation 

172.16.229.128    172.16.229.129
+------------+     +------------+
|Hyperspace  |     |spare ?!    |
|Master      |     |            |
|Cap Source  |     |            |
|------------|     |------------|    
|Namenode    |     |Secondary   |
|            |     |Namenode &  |
|            |     |JobTracker  |
|------------|     |------------|    
|nn01        |     |jt01        |
+-----+------+     +------+-----+            +----NAT(@VM)
      |                  |                  |
------+------------------+------------------+
      |                  |                  |
+-----+------+     +------+-----+     +------+-----+
|RangeServer |     |RangeServer |     |RangeServer |
|ThriftBroker|     |ThriftBroker|     |ThriftBroker|
|------------|     |------------|     |------------|
|Datanode    |     |Datanode    |     |Datanode    |
|TaskTracker |     |TaskTracker |     |TaskTracker |
|------------|     |------------|     |------------|
|dn01        |     |dn02        |     |dn02        |
+------------+     +------------+     +------------+
172.16.229.130     172.16.229.131     172.16.229.132


1. password-less ssh
2. /etc/hosts
#note hypertable master should refer to hadoop hostname
## Master       ==> nn01
## Hypersapce   ==> nn01
## Cap Source   ==> nn01
## <none>       ==> jt01   #maybe spare master
## RangeServer  ==> dn01 ~ dn03
## ThriftBroker ==> dn01 ~ dn03
[root@nn01 yum.repos.d]# cat /etc/hosts
172.16.229.128 hd01 nn01 namenode nn01.hd
172.16.229.129 hd02 jt01 jobtracker  nn02 #secondary namenode
172.16.229.130 hd03 dn01 datanode01
172.16.229.131 hd04 dn02 datanode02
172.16.229.132 hd05 dn03 datanode03

3. allow simultaneous connections from all of the machines in the Hypertable cluster(@admin or @source)
[root@nn01 ~]# vi /etc/ssh/sshd_config
  :
#MaxStartups 10
MaxStartups 100
  :
[root@nn01 ~]# service sshd restart

4. turn off firewall
   or
   maintance iptables for hypertable cluster

5. config open files limits and swappiness(@ all hosts)
vi /etc/security/limits.conf
  :
* - nofile 65536
  :
# or
  :
hdfs - nofile 65536        #Hadoop processes owned by hdfs(@ hadoop cluster ?!)
hypertable - nofile 65536    #Hypertable processes owned by hypertable

#and then re-login , or reboot

[root@nn01 ~]# ulimit -a
  :
open files                      (-n) 65536
  :

[root@nn01 ~]# vi /etc/sysctl.conf
  :
vm.swappiness=0

[root@nn01 ~]# sysctl -p

6.Gem , Mornitor and Capistrano
##install gem @nn01
[root@nn01 ~]# yum install git-core ruby ruby-devel ruby-ext ruby-rdoc
[root@nn01 ~]# wget http://rubyforge.org/frs/download.php/75475/rubygems-1.8.11.tgz
[root@nn01 ~]# tar -zxvf rubygems-1.8.11.tgz
[root@nn01 ~]# cd rubygems-1.8.11;ruby setup.rb
## install capistrano @nn01
[root@nn01 rubygems-1.8.11]# gem install capistrano

##install tools @nn01 (for monitoring server setup)
[root@nn01 rubygems-1.8.11]# yum install gcc-c++ -y
[root@nn01 rubygems-1.8.11]# gem install sinatra rack thin json titleize
[root@nn01 rubygems-1.8.11]# yum install rrdtool -y
[root@nn01 ~]# /sbin/ldconfig

[root@nn01 rubygems-1.8.11]# cd ~

#copy sample file from other
[root@nn01 ~]# scp HT243:~/Cap* .
Capfile.cluster                               100%   12KB  11.9KB/s   00:00   

[root@nn01 ~]# vi Capfile.cluster
set :source_machine,     "nn01"
set :install_dir,        "/opt/hypertable"
set :hypertable_version, "0.9.6.1"
set :default_pkg,        "/extras/hypertable-0.9.6.1-linux-x86_64.rpm"
set :default_dfs,        "hadoop"
set :default_config,     "/root/hypertable.cfg"

role :source, "nn01"
role :master, "nn01"
role :hyperspace, "nn01"
role :slave,  "dn01", "dn02", "dn03"
role :localhost, "nn01"
role :thriftbroker
role :spare, "jt01"

## note: don't use difference hostname (or alias) for the same machine , otherwise you will got error

[root@nn01 ~]# mv Capfile.cluster Capfile
[root@nn01 ~]# mkdir /extras
[root@nn01 ~]# cd /extras
[root@nn01 extras]# wget ftp://HT248/extras/hypertable*
[root@nn01 extras]# ls -l /extras/hypertable-0.9.6.1-linux-x86_64.rpm
-rw-r--r-- 1 root root 94864139 Aug 24 00:00 /extras/hypertable-0.9.6.1-linux-x86_64.rpm


7. installation
# dependence packages you must install for whole cluster's servers
## for /opt/hypertable/current/bin/jrn
[root@nn01 ~]# yum install java-1.6.0-openjdk java-1.6.0-openjdk-devel -y
## for hypertable rpm dependency
[root@nn01 ~]# yum install perl-Bit-Vector perl-IO-String perl-IO-Zlib perl-IO-Socket-INET6 perl-IO-Socket-SSL perl-libwww-perl perl-HTTP-Request

[root@nn01 extras]# cd ~
[root@nn01 ~]# cap install_package
[root@nn01 ~]# cap dist
# FHS-ize: http://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard
[root@nn01 ~]# cap shell
cap> mkdir /etc/opt/hypertable /var/opt/hypertable
cap> chown root:root /etc/opt/hypertable /var/opt/hypertable
cap> exit
exiting
[root@nn01 ~]# cap fhsize


8. startup
##mkdir for hypertable (with hadoop command)
[root@nn01 ~]# sudo -u hdfs hadoop fs -mkdir /hypertable
[root@nn01 ~]# sudo -u hdfs hadoop fs -ls /
Found 6 items
drwxr-xr-x   - hdfs supergroup          0 2012-08-24 16:38 /hypertable
drwxr-xr-x   - root supergroup          0 2012-08-23 02:56 /hypertable-solo
drwxr-xr-x   - hdfs supergroup          0 2012-08-24 12:04 /root
drwxrwxrwt   - hdfs supergroup          0 2012-08-24 09:11 /tmp
drwxr-xr-x   - hdfs supergroup          0 2012-08-23 02:00 /user
drwxr-xr-x   - hdfs supergroup          0 2012-08-24 12:08 /usr
[root@nn01 ~]# sudo -u hdfs hadoop fs -chown -R root /hypertable
[root@nn01 ~]# sudo -u hdfs hadoop fs -ls /
Found 6 items
drwxr-xr-x   - root supergroup          0 2012-08-24 16:38 /hypertable
drwxr-xr-x   - root supergroup          0 2012-08-23 02:56 /hypertable-solo
drwxr-xr-x   - hdfs supergroup          0 2012-08-24 12:04 /root
drwxrwxrwt   - hdfs supergroup          0 2012-08-24 09:11 /tmp
drwxr-xr-x   - hdfs supergroup          0 2012-08-23 02:00 /user
drwxr-xr-x   - hdfs supergroup          0 2012-08-24 12:08 /usr

[root@nn01 ~]# scp HT248:~/hypertable.cfg ~/.
hypertable.cfg                                100%  633     0.6KB/s   00:00   

[root@nn01 ~]# vi ~/hypertable.cfg
  :
HdfsBroker.fs.default.name=hdfs://nn01:9000/
  :
# DFS Broker - for clients
DfsBroker.Port=38030
#don't just set as nn01 , or all slaves will not be excuted dfsbroker
DfsBroker.Host=localhost 
  :
# Hyperspace
Hyperspace.Replica.Host=nn01
#or --> Hyperspace.Replica.Host=localhost
  :

[root@nn01 ~]# cap push_config
[root@nn01 ~]# cap set_current
[root@nn01 ~]# cap shell
  * executing `shell'
====================================================================
Welcome to the interactive Capistrano shell! This is an experimental
feature, and is liable to change in future releases. Type 'help' for
a summary of how to use the shell.
--------------------------------------------------------------------
cap> date
[establishing connection(s) to dn01, dn02, dn03, nn01, jt01]
 ** [out :: dn01] Fri Aug 24 12:19:21 CST 2012
 ** [out :: dn02] Fri Aug 24 12:19:21 CST 2012
 ** [out :: dn03] Fri Aug 24 12:19:21 CST 2012
 ** [out :: nn01] Fri Aug 24 12:19:21 CST 2012
 ** [out :: jt01] Fri Aug 24 12:19:21 CST 2012
cap> quit
exiting
## you could make a line for excuting it instead, like
## cap invoke COMMAND="date"

[root@nn01 ~]# cap start

##you could close the cluster as
[root@nn01 ~]# cap stop

[root@nn01 log]# elinks -dump http://nn01:38090/
   [1]Hypertable

                             Hypertable Monitoring

   [2]Range Servers  | [3]Tables

  Range Servers

    Number of Range Servers: 3

    Master Version: Python 2.6.8

                                                                            Disk Disk RAM  Clock Range Last    Last
Server[4][IMG] Host       IP       System        Kernel         Arch  Cores (TB) Use  (GB) Skew  Count Error Contact
                                                                                 (%)
                                                                                                             Wed Aug
[5]rs1         dn03 172.16.229.132 CentOS Linux                x86_64     1 0.20    1 1.02 0     4     ok    29
                                   6.0    2.6.32-71.el6.x86_64                                               15:14:14
                                                                                                             2012
                                                                                                             Wed Aug
[6]rs2         dn02 172.16.229.131 CentOS Linux                x86_64     1 0.20    1 1.02 0     2     ok    29
                                   6.0    2.6.32-71.el6.x86_64                                               15:14:14
                                                                                                             2012
                                                                                                             Wed Aug
[7]rs3         dn01 172.16.229.130 CentOS Linux                x86_64     1 0.20    1 1.02 0     2     ok    29
                                   6.0    2.6.32-71.el6.x86_64                                               15:14:14
                                                                                                             2012

   Hypertable

References

   Visible links
   2. http://nn01:38090/
   3. http://nn01:38090/tables
   4. http://nn01:38090/?sort=name&ord=dsc
   5. http://nn01:38090/graphs?server=rs1
   6. http://nn01:38090/graphs?server=rs2
   7. http://nn01:38090/graphs?server=rs3


[FAQ]
(1) cap XXX
   You must executed "$cap XXX" at the folder with "Capfile". In this deployment,
   Capfile is exist at /root . That means all "cap XXX" must change your PWD to /root.
   It also means , It could only launch hypertable by 'root' user.
(2) Hypertable should be routable with all nodes of Hadoop
   Hypertable's DfsBroker(hadoop Broker) is as a DFS client . When it Read/Write on HDFS,
   1) Get File Pos(Metadata) from Namenode ,and tell DFS client where the file is.
   2) Go to Datanode, and Read/Write for the file at Pos.
   Otherwise , when you start Hypertable.Master up . It will give you errors.
(3) Master Could not start up
   when Master could not start up , and you got a message in DfsBroker.hadoop.log

Aug 29, 2012 2:29:10 PM org.hypertable.DfsBroker.hadoop.HdfsBroker Create
SEVERE: I/O exception while creating file '/hypertable/servers/master/log/mml/1' - org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hdfs.server.namenode.SafeModeException: Cannot create file/hypertable/servers/master/log/mml/1. Name node is in safe mode.
   
   try to 'cap stop' and 'cap start'
      or
  'cap stop' ; remove /hypertable/servers/master/log/mml/## ; 'cap start'

   maybe the problem is no longer exist.

reference:
http://code.google.com/p/hypertable/wiki/HypertableManual
https://groups.google.com/forum/?fromgroups#!topic/hypertable-user/a7bMbucORMg
http://hypertable.com/documentation/administrator_guide/monitoring_system/
https://groups.google.com/forum/?fromgroups#!topic/hypertable-user/ZcioX6bUKd4
http://osdir.com/ml/hypertable-user/2010-06/msg00021.html

2012年8月22日 星期三

handout for hadoop cluster installation and deployment

172.16.229.128    172.16.229.129
+-----------+     +------------+
|Namenode   |     |Secondary   |
|nn01       |     |Namenode &  |
|           |     |JobTracker  |
|           |     |jt01        |
+-----+-----+     +------+-----+            +----NAT(@VM)
      |                  |                  |
------+------------------+------------------+
      |                  |                  |
+-----+-----+     +------+-----+     +------+-----+
|Datanode   |     |Datanode    |     |Datanode    |
|   &       |     |      &     |     |      &     |
|TaskTracker|     |TaskTracker |     |TaskTracker |
|dn01       |     |dn02        |     |dn02        |
+-----------+     +------------+     +------------+
172.16.229.130    172.16.229.131     172.16.229.132

Environment: (@VM)

        host : CentOS-6.0
        kernel: 2.6.32-71.el6.x86_64 #1 SMP
        MemTotal: 7891424 kB(/proc/meminfo)
        CPU(Quad core) : Intel(R) Core(TM) i3-2130 CPU @ 3.40GHz
        HDD: 1TB*2 (LVM ==> 2T)

        guest: CentOS-6.0
        kernel: 2.6.32-71.el6.x86_64 #1 SMP
        MemTotal: 1G
        CPU *1
        HDD: 200GB

@NFS Server(VM host)

yum install -y rpcbind nfs-utils
mkdir /home/nfs
vi /etc/exports
  /home/nfs         172.16.229.0/24(rw,sync,no_root_squash) *(ro,sync,no_root_squash)
chmod -R 777 /home/nfs
#nfs permission options may include 'no_root_squash',
#or you would get error when you format your namenode(sudo -u hdfs hadoop namenode -format)
#some error when you lsof: (/nfsmount/dfs/nn/current/nfs....... -> java will be stuck by the file)
#no enough permissin for hdfs


chkconfig rpcbind on
chkconfig nfs on
chkconfig nfslock on
service rpcbind start
service nfs start
service nfslock start

showmount -e localhost
  Export list for localhost:
  /home/nfs 172.16.229.0/24,localhost.localdomain

mkdir /mnt/test
mount -t nfs -o tcp,soft,intr,timeo=10,retrans=10 172.16.229.1:/home/nfs /mnt/test
umount /mnt/test


@all hadoop/hdfs nodes(VM guests)

[root@nn01 yum.repos.d]# cat /etc/hosts
172.16.43.248   HT248   # Added by NetworkManager
127.0.0.1       localhost.localdomain   localhost
::1     HT248   localhost6.localdomain6 localhost6
172.16.43.248 HT248
172.16.229.128 hd01 nn01 namenode nn01.hd
172.16.229.129 hd02 jt01 jobtracker  nn02 #secondary namenode
172.16.229.130 hd03 dn01 datanode01
172.16.229.131 hd04 dn02 datanode02
172.16.229.132 hd05 dn03 datanode03

[root@nn01 ~]# scp /etc/hosts jt01:/etc/.
[root@nn01 ~]# scp /etc/hosts dn01:/etc/.
[root@nn01 ~]# scp /etc/hosts dn02:/etc/.
[root@nn01 ~]# scp /etc/hosts dn03:/etc/.


ssh password-less for all hosts


#got cdh3 gpg key for all hosts
rpm --import http://archive.cloudera.com/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera

#add a repository of cdh3 for all hosts
[cloudera-cdh3]
name=Cloudera's Distribution for Hadoop, Version 3
mirrorlist=http://archive.cloudera.com/redhat/6/x86_64/cdh/3/mirrors
gpgkey = http://archive.cloudera.com/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera   
gpgcheck = 1

#install hadoop and hadoop-native for all hosts
yum install hadoop-0.20 hadoop-0.20-native -y
yum install hadoop-0.20-namenode hadoop-0.20-datanode hadoop-0.20-secondarynamenode hadoop-0.20-jobtracker hadoop-0.20-tasktracker nfs-utils -y


[root@nn01 ~]# yum install java-1.6.0-openjdk java-1.6.0-openjdk-devel -y
[root@nn01 ~]# vi /usr/lib/hadoop-0.20/conf/hadoop-env.sh
# Set Hadoop-specific environment variables here.

# The only required environment variable is JAVA_HOME.  All others are
# optional.  When running a distributed configuration it is best to
# set JAVA_HOME in this file, so that it is correctly defined on
# remote nodes.

# The java implementation to use.  Required.
# export JAVA_HOME=/usr/lib/j2sdk1.6-sun
export JAVA_HOME=/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64

[root@nn01 ~]# vi /etc/profile
  :
export JAVA_HOME=/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64

[root@nn01 ~]# scp /usr/lib/hadoop-0.20/conf/hadoop-env.sh jt01:/usr/lib/hadoop-0.20/conf/.
[root@nn01 ~]# scp /usr/lib/hadoop-0.20/conf/hadoop-env.sh dn01:/usr/lib/hadoop-0.20/conf/.
[root@nn01 ~]# scp /usr/lib/hadoop-0.20/conf/hadoop-env.sh dn02:/usr/lib/hadoop-0.20/conf/.
[root@nn01 ~]# scp /usr/lib/hadoop-0.20/conf/hadoop-env.sh dn03:/usr/lib/hadoop-0.20/conf/.
[root@nn01 ~]# scp /etc/profile jt01:/etc/.
[root@nn01 ~]# scp /etc/profile dn01:/etc/.
[root@nn01 ~]# scp /etc/profile dn02:/etc/.
[root@nn01 ~]# scp /etc/profile dn03:/etc/.

##config open files limits(@ all hosts)
vi /etc/security/limits.conf
  :
* - nofile 65536
  :
# or
  :
hdfs - nofile 65536        #Hadoop processes owned by hdfs(@ hadoop cluster ?!)
hypertable - nofile 65536    #Hypertable processes owned by hypertable

#and then re-login , or reboot

[root@nn01 ~]# ulimit -a
  :
open files                      (-n) 65536
  :


#close all hadoop service for all hosts, before hdfs was installed successful.
chkconfig hadoop-0.20-namenode off
chkconfig hadoop-0.20-jobtracker off
chkconfig hadoop-0.20-secondarynamenode off
chkconfig hadoop-0.20-tasktracker off
chkconfig hadoop-0.20-datanode off

#@nn01
chkconfig rpcbind on
chkconfig nfslock on
service rpcbind start
service nfslock start

#mkdir for all hosts
##Create the dfs.name.dir local directories
mkdir -p /data/1/dfs/nn /nfsmount/dfs/nn

##Create the dfs.data.dir local directories(as volumes of datanode)
mkdir -p /data/1/dfs/dn /data/2/dfs/dn /data/3/dfs/dn /data/4/dfs/dn

##Create the mapred.local.dir local directories
mkdir -p /data/1/mapred/local /data/2/mapred/local /data/3/mapred/local /data/4/mapred/local

##owner and permission config
chown -R hdfs:hadoop /data/1/dfs/nn /nfsmount/dfs/nn /data/1/dfs/dn /data/2/dfs/dn /data/3/dfs/dn /data/4/dfs/dn
chown -R mapred:hadoop /data/1/mapred/local /data/2/mapred/local /data/3/mapred/local /data/4/mapred/local
chmod -R 700 /data/1/dfs/nn /nfsmount/dfs/nn
chmod -R 700 /data/1/dfs/dn /data/2/dfs/dn /data/3/dfs/dn /data/4/dfs/dn
chmod -R 755 /data/1/mapred/local /data/2/mapred/local /data/3/mapred/local /data/4/mapred/local


#mount nfs(CentOS6) from VM host(only @nn01)
mount -t nfs -o tcp,soft,intr,timeo=10,retrans=10 172.16.229.1:/home/nfs /nfsmount/dfs/nn
umount /nfsmount/dfs/nn
vi /etc/fstab
  172.16.229.1:/home/nfs    /nfsmount/dfs/nn    nfs tcp,soft,intr,timeo=10,retrans=10 1 1
mount /nfsmount/dfs/nn

#custumizing config for all hosts : make a copy of "empty" configuration
cp -r  /etc/hadoop-0.20/conf.empty /etc/hadoop-0.20/conf.my_cluster

#custumizing config for all hosts : make a higher priority of the config
alternatives --install /etc/hadoop-0.20/conf hadoop-0.20-conf /etc/hadoop-0.20/conf.my_cluster 50

alternatives --display hadoop-0.20-conf

#custumizing config for all hosts : core-site.xml(/etc/hadoop/conf.my_cluster/core-site.xml)
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
        <property>
            <name>fs.default.name</name>
            <value>hdfs://nn01:9000</value>
         </property>
    </configuration>

#custumizing config for hdfs-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
        <property>
            <name>dfs.name.dir</name>
            <value>/data/1/dfs/nn,/nfsmount/dfs/nn</value>
        </property>
        <property>
            <name>dfs.data.dir</name>
            <value>/data/1/dfs/dn,/data/2/dfs/dn,/data/3/dfs/dn</value>
        </property>
        <property>
            <name>dfs.webhdfs.enabled</name>
            <value>true</value>
        </property>
        <property>
            <name>dfs.datanode.failed.volumes.tolerated</name>
            <value>2</value>
        </property>
        <property>
            <name>dfs.http.address</name>
            <value>nn01:50070</value>
        </property>

    </configuration>

#custumizing config for  mapred-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
        <property>
            <name>mapred.job.tracker</name>
            <value>jt01:9001</value>
        </property>
        <property>
            <name>mapred.local.dir</name>
            <value>/data/1/mapred/local,/data/2/mapred/local,/data/3/mapred/local</value>
        </property>
        <property>
            <name>mapreduce.jobtracker.restart.recover</name>
            <value>true</value>
        </property>

    </configuration>

#custumizing config for masters and slaves
##/etc/hadoop/conf.my_cluster/masters, jt01 for Secondary Namenode
nn01
jt01

##/etc/hadoop/conf.my_cluster/slaves
dn01
dn02
dn03

#deployment all nodes
[root@nn01 ~]# scp -r /etc/hadoop/conf.my_cluster jt01:/etc/hadoop/.
[root@nn01 ~]# scp -r /etc/hadoop/conf.my_cluster dn01:/etc/hadoop/.
[root@nn01 ~]# scp -r /etc/hadoop/conf.my_cluster dn02:/etc/hadoop/.
[root@nn01 ~]# scp -r /etc/hadoop/conf.my_cluster dn03:/etc/hadoop/.


##Check conf priority in all hosts
alternatives --display hadoop-0.20-conf
### /etc/hadoop-0.20/conf.my_cluster is suppose to  50(the highest)

#format namenode @nn01
[root@nn01 ~]# sudo -u hdfs hadoop namenode -format
12/08/22 19:05:42 INFO namenode.NameNode: STARTUP_MSG:
/************************************************************
STARTUP_MSG: Starting NameNode
STARTUP_MSG:   host = nn01/172.16.229.128
STARTUP_MSG:   args = [-format]
STARTUP_MSG:   version = 0.20.2-cdh3u5
STARTUP_MSG:   build = file:///data/1/tmp/topdir/BUILD/hadoop-0.20.2-cdh3u5 -r 580d1d26c7ad6a7c6ba72950d8605e2c6fbc96cc; compiled by 'root' on Mon Aug  6 20:22:48 PDT 2012
************************************************************/
Re-format filesystem in /data/1/dfs/nn ? (Y or N) Y
Re-format filesystem in /nfsmount/dfs/nn ? (Y or N) Y
12/08/22 19:08:35 INFO util.GSet: VM type       = 64-bit
12/08/22 19:08:35 INFO util.GSet: 2% max memory = 19.33375 MB
12/08/22 19:08:35 INFO util.GSet: capacity      = 2^21 = 2097152 entries
  :
  :
  :
## UPPER-CASE 'Y' is neccessary when you would like to re-format namenode.



#####NOTE: if you got error like
+======================================================================+
|      Error: JAVA_HOME is not set and Java could not be found         |
+----------------------------------------------------------------------+
| Please download the latest Sun JDK from the Sun Java web si/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64te        |
|       > http://java.sun.com/javase/downloads/ <                      |
|                                                                      |
| Hadoop requires Java 1.6 or later.                                   |
| NOTE: This script will find Sun Java whether you install using the   |
|       binary or the RPM based installer.                             |
+======================================================================+

##check your $JAVA_HOME is set in /usr/lib/hadoop-0.20/conf/hadoop-env.sh

#start hdfs service right now and auto-start when reboot
## Namenode@nn01
[root@nn01 ~]# service hadoop-0.20-namenode start
[root@nn01 ~]# chkconfig hadoop-0.20-namenode on

## Secondary Namenode @jt01
[root@jt01 ~]# service hadoop-0.20-secondarynamenode start
[root@jt01 ~]# chkconfig hadoop-0.20-secondarynamenode on

## DataNode @dn01~dn03
[root@dn01 ~]# service hadoop-0.20-datanode start
[root@dn02 ~]# service hadoop-0.20-datanode start
[root@dn03 ~]# service hadoop-0.20-datanode start
[root@dn01 ~]# chkconfig hadoop-0.20-datanode on
[root@dn02 ~]# chkconfig hadoop-0.20-datanode on
[root@dn03 ~]# chkconfig hadoop-0.20-datanode on

#Testing around nodes
[root@nn01 ~]# sudo -u hdfs hadoop fs -mkdir /tmp
[root@jt01 ~]# sudo -u hdfs hadoop fs -chmod -R 1777 /tmp
[root@dn03 ~]# sudo -u hdfs hadoop fs -ls /
Found 1 items
drwxrwxrwt   - hdfs supergroup          0 2012-08-22 23:02 /tmp
##watch out file permission inside hdfs, serveral user need with Write-access Permission
## such as : hdfs / root / mapred ... 


#start MapReduce service right now and auto-start when reboot
## JobTracker @jt01
[root@jt01 ~]# service hadoop-0.20-jobtracker start
[root@jt01 ~]# chkconfig hadoop-0.20-jobtracker on

## TaskTracker @dn01~dn03
[root@dn01 ~]# chkconfig hadoop-0.20-tasktracker on
[root@dn02 ~]# chkconfig hadoop-0.20-tasktracker on
[root@dn03 ~]# chkconfig hadoop-0.20-tasktracker on
[root@dn01 ~]# service hadoop-0.20-tasktracker start
[root@dn02 ~]# service hadoop-0.20-tasktracker start
[root@dn03 ~]# service hadoop-0.20-tasktracker start

#After all hosts rebooted

[root@nn01 ~]# netstat -ptlnu | grep java
tcp        0      0 172.16.229.128:9000         0.0.0.0:*                   LISTEN      1486/java          
tcp        0      0 172.16.229.128:50070        0.0.0.0:*                   LISTEN      1486/java          
tcp        0      0 0.0.0.0:60551               0.0.0.0:*                   LISTEN      1486/java          

[root@jt01 ~]# netstat -ptlnu | grep java
tcp        0      0 172.16.229.129:9001         0.0.0.0:*                   LISTEN      1387/java          
tcp        0      0 0.0.0.0:50090               0.0.0.0:*                   LISTEN      1468/java          
tcp        0      0 0.0.0.0:34733               0.0.0.0:*                   LISTEN      1387/java          
tcp        0      0 0.0.0.0:50030               0.0.0.0:*                   LISTEN      1387/java          
tcp        0      0 0.0.0.0:59731               0.0.0.0:*                   LISTEN      1468/java          

[root@dn01 ~]# netstat -ptlnu | grep java
tcp        0      0 0.0.0.0:50060               0.0.0.0:*                   LISTEN      1515/java          
tcp        0      0 0.0.0.0:50010               0.0.0.0:*                   LISTEN      1386/java          
tcp        0      0 0.0.0.0:55162               0.0.0.0:*                   LISTEN      1386/java          
tcp        0      0 0.0.0.0:50075               0.0.0.0:*                   LISTEN      1386/java          
tcp        0      0 0.0.0.0:50020               0.0.0.0:*                   LISTEN      1386/java          
tcp        0      0 127.0.0.1:45221             0.0.0.0:*                   LISTEN      1515/java          

[root@dn02 ~]# netstat -ptlnu | grep java
tcp        0      0 0.0.0.0:50060               0.0.0.0:*                   LISTEN      1528/java          
tcp        0      0 0.0.0.0:47663               0.0.0.0:*                   LISTEN      1399/java          
tcp        0      0 0.0.0.0:50010               0.0.0.0:*                   LISTEN      1399/java          
tcp        0      0 127.0.0.1:55771             0.0.0.0:*                   LISTEN      1528/java          
tcp        0      0 0.0.0.0:50075               0.0.0.0:*                   LISTEN      1399/java          
tcp        0      0 0.0.0.0:50020               0.0.0.0:*                   LISTEN      1399/java          

[FAQ]

(1) namenode start-up error
  PriviledgedActionException as:hdfs (auth:SIMPLE) cause:java.net.BindException:

  checking hdfs-site.xml and masters
  if secondary-name node has been assigned , you should list it into masters

(2) for more performance, modify /etc/sysctl.conf
  vi /etc/sysctl.conf
    vm.swappiness=0
  sysctl -p
  #for more memory usage( not mention of zero-swap usage)
(3) Secodary Namenode setting:
  1) hdfs-site.xml@SecondaryNameNode(or all hosts) should contain:
        <property>
            <name>dfs.http.address</name>
            <value>nn01:50070</value>
        </property>
      #nn01 is namenode
  2) masters(all hosts) should contain:
     nn01
     jt01
     #Namenode and SecondaryNameNode
  3) startup namenode service @nn01
  4) startup secondarynamenode service @jt01
  Note: (refer to https://ccp.cloudera.com/display/CDHDOC/CDH3+Deployment+on+a+Cluster)
The Secondary NameNode actually performs an HTTP GET request to retrieve the current fsimage, (checkpoint), and edits log from the NameNode and the HTTP PUT request to upload the new checkpoint back to the NameNode. The dfs.http.address property defines the mask and host where the NameNode listens for the Secondary NameNode to connect.

2012年8月20日 星期一

handout about building hypertable-0.9.6.1 (tarball)

Evironment:
       host : CentOS-6.0
       kernel: 2.6.32-71.el6.x86_64 #1 SMP
       MemTotal:        7891424 kB(/proc/meminfo)
       CPU(Quad core)      : Intel(R) Core(TM) i3-2130 CPU @ 3.40GHz
       HDD: 1TB*2 (LVM ==> 2T)
       

       guest: CentOS-6.0
       kernel: 2.6.32-71.el6.x86_64 #1 SMP
       MemTotal:        1G
       CPU *1
       HDD: 100GB 

yum install gcc gcc-c++ make ntpdate cmake java-1.6.0-openjdk java-1.6.0-openjdk-devel
yum install icu libicu libicu-devel   #for libboost_regex.so
rpm -Uvh http://ftp.sjtu.edu.cn/sites/download.fedora.redhat.com/pub/epel/6/x86_64/epel-release-6-7.noarch.rpm
yum install log4cpp log4cpp-devel expat expat-devel uuid uuid-devel libuuid libuuid-devel python python-devel fuse fuse-libs fuse-devel openssl openssl-devel ant zlib-devel

rpm -ivh xfsprogs-3.1.1-7.el6.x86_64.rpm xfsprogs-devel-3.1.1-7.el6.x86_64.rpm

yum install sigar sigar-devel bzip2 bzip2-devel rrdtool libevent-devel cronolog libedit-devel readline readline-devel
#if you need jemalloc jemalloc-devel , then install it via yum instal...

/usr/sbin/ntpdate -b time.stdtime.gov.tw;/sbin/hwclock --systohc --utc

cd /usr/local
wget http://hi.baidu.com/cpuramdisk/item/3410d0ccc53e3f0b0ad93a00
#or
#wget http://download.oracle.com/berkeley-db/db-4.8.26.tar.gz
tar -zxvf db-4.8.26.tar.gz
cd db-4.7.25/build_unix/
../dist/configure --enable-cxx (--enable-java;)(root permission is neccessary)
make
make install
echo “/usr/local/BerkeleyDB.4.8/lib” >> /etc/ld.so.conf.d /BerkeleyDB.conf
sudo /sbin/ldconfig

#I had been installed for boost-1.41.x via yum, but hypertable may not be built successful, an error
#hypertable struct std::basic_string<char, std::char_traits<char>, std::allocator<char> >
#when you re-install boost_1_49_0 as below, you may need to re-build kfs too.
#I recommend you remove /usr/local/kfs-0.5/build , and re-build it . Otherwise, you will get errors.
#That errors will be something link "libboost_regex-mt.so ..."
cd /usr/local
wget http://nchc.dl.sourceforge.net/project/boost/boost/1.49.0/boost_1_49_0.tar.gz
tar -zxvf boost_1_49_0.tar.gz
cd boost_1_49_0/tools/build/v2/
./bootstrap.sh
./b2 install --prefix=/usr/local
cd /usr/local/boost_1_49_0/
./bootstrap.sh --with-icu --with-libraries=regex,filesystem,chrono,date_time,exception,iostreams,locale,program_options,python,random,serialization,signals,system,test,thread,timer,wave
./b2
./b2 install
ln -s /usr/local/boost_1_49_0/stage/lib/libboost_regex.a /usr/lib64/libboost_regex.a
ln -s /usr/local/boost_1_49_0/stage/lib/libboost_regex.so /usr/lib64/libboost_regex.so
ln -s /usr/local/boost_1_49_0/stage/lib/libboost_regex.so.1.49.0 /usr/lib64/libboost_regex.so.1.49.0


cd /usr/local
wget http://ftp.heanet.ie/mirrors/sourceforge/k/project/ko/kosmosfs/kosmosfs/kfs-0.5/kfs-0.5.tar.gz
tar -zxvf kfs-0.5.tar.gz
mkdir -p kfs-0.5/build
cd kfs-0.5/build
vi ../CMakeLists.txt
#line 110(after  'set (CMAKE_BUILD_TYPE "Debug")' ) : add 2 lines java setting
    set (JAVA_INCLUDE_PATH "/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64/include")
   set (JAVA_INCLUDE_PATH2 "/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64/include")

#line 137: fill fuse setting
SET(Fuse_LIBRARY_DIR "/lib64")
SET(Fuse_INCLUDE_DIR "/usr/include")

cmake -D CMAKE_BUILD_TYPE=RelWithDebInfo /usr/local/kfs-0.5
gmake
gmake install

cd ..
ant jar

vi ~/.bashrc
export CLASSPATH=${CLASSPATH}:/usr/local/kfs-0.5/build/kfs-0.5.jar

cd /usr/local/kfs-0.5/src/cc/access
vi kfs_setup.py
kfsext = Extension('kfs',
                include_dirs = ['/usr/local/kfs-0.5/src/cc/', '/usr/local/include/boost/'],
## use full path name in it.

python kfs_setup.py /usr/local/kfs-0.5/build/lib/ build
python kfs_setup.py /usr/local/kfs-0.5/build/lib/ install
python kfs_setup.py /usr/local/kfs-0.5/build/lib/ install --home=/usr/local/kfs-0.5/build/lib


vi ~/.bashrc
export PYTHONPATH=${PYTHONPATH}:/usr/local/kfs-0.5/build/lib/lib64/python
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/kfs-0.5/build/lib


wget http://mirror.yandex.ru/mandriva/devel/cooker/SRPMS/contrib/release/re2-0-1.src.rpm
rpm -ivh re2-0-1.src.rpm
cd ~/rpmbuild/SOURCES/
tar -zxvf re2.tar.gz
cd re2
make;make install

cd /usr/local
wget http://snappy.googlecode.com/files/snappy-1.0.4.tar.gz
tar -zxvf snappy-1.0.4.tar.gz
cd snappy-1.0.4
./configure; make; make install

#<optional> if you don't need ceph file system, skip this step
cd /usr/local
wget http://ceph.newdream.net/download/ceph-0.22.2.tar.gz
tar xzvf ceph-0.49.tar.gz
cd ceph-0.49
./configure; make; make install

cd /usr/local
wget http://archive.apache.org/dist/thrift/0.8.0/thrift-0.8.0.tar.gz
tar -zxvf thrift-0.8.0.tar.gz
cd  thrift-0.8.0
./configure
make
make install
/sbin/ldconfig

wget http://magelan.googlecode.com/files/hyperic-sigar-1.6.4.zip
unzip hyperic-sigar-1.6.4.zip
cp hyperic-sigar-1.6.4/sigar-bin/include/*.h /usr/local/include
cp hyperic-sigar-1.6.4/sigar-bin/lib/libsigar-amd64-linux.so /usr/local/lib


echo "/usr/local/lib" >> /etc/ld.so.conf.d/normal.conf
/sbin/ldconfig


cd /usr/local
http://hypertable.googlecode.com/files/hypertable-0.9.6.1-src.tar.gz
tar -zxvf hypertable-0.9.6.1-src.tar.gz
cd hypertable-0.9.6.1

#if you don't wanna kosmosBroker, skip this step
vi CMakeLists.txt
#update "find_package(Kfs)" into
find_package(Kfs REQUIRED)

vi cmake/FindKfs.cmake
#Add a line about Line 33: (after "find_path(Kfs_INCLUDE_DIR kfs/KfsClient.h "...)
  /usr/local/kfs-0.5/build/include

#Update  Line 42:  (after  "find_library(${lib}_LIB NAMES ${lib}")
#before
  find_library(${lib}_LIB NAMES ${lib}
    PATHS /opt/kfs/lib/static /opt/kfs/lib /opt/local/lib /usr/local/lib
#after
  find_library(${lib}_LIB NAMES ${lib}
    PATHS /opt/kfs/lib/static /opt/kfs/lib /opt/local/lib /usr/local/lib /usr/local/kfs-0.5/build/lib

#Add a line about Line 54: (after "find_library(Crypto_LIB "...)
find_library(regex_LIB NAMES boost_regex PATHS /usr/local/lib)

#Update  Line 59:  (after  "if (Kfs_INCLUDE_DIR AND kfsClient_LIB)...")
#before
if (Kfs_INCLUDE_DIR AND kfsClient_LIB)
  set(Kfs_FOUND TRUE)
  set( Kfs_LIBRARIES ${kfsClient_LIB} ${kfsIO_LIB} ${kfsCommon_LIB}
                     ${qcdio_LIB} ${Crypto_LIB})

#after
if (Kfs_INCLUDE_DIR AND kfsClient_LIB)
  set(Kfs_FOUND TRUE)
  set( Kfs_LIBRARIES ${kfsClient_LIB} ${kfsIO_LIB} ${kfsCommon_LIB}
                     ${qcdio_LIB} ${Crypto_LIB} ${regex_LIB})

mkdir build
cd build
cmake ../
gmake
gmake install
#all bin file default will copy to /opt/hypertable/.

--------------------------KFS Deployment----------------------------------------------------


ssh-keygen -t dsa
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

cd /usr/local/kfs-0.5/scripts

vi ../webui/all-machines.txt
HT248

vi machines.cfg
# KFS Machine configuration file
[metaserver]
node: HT248
clusterkey: kfs
rundir: /home/kfs/meta
baseport: 20000
[chunkserver1]
node: HT248
rundir: /home/kfs/chunk1
baseport: 30000
space: 200 G
[chunkserver2]
node: HT248
rundir: /home/kfs/chunk2
baseport: 40000
space: 200 G


#single node
python kfssetup.py -f machines.cfg -b ../build -w ../webui -s

#multiple nodes
python kfssetup.py -f machines.cfg  -b ../build -w ../webui

#startup

python kfslaunch.py -f machines.cfg  -s
# after startup kfs , if webui didn't listen 20050 port . you should :
# (1) stop kfs
# (2) remove /home/kfs directory
# (3) re-setup kfs (python kfssetup.py -f machines.cfg ...)
# (4) check server.conf and all-machines.txt are exist at /home/kfs/meta/webui/
# (5) if webui config is ok , startup kfs . you could elinks to 20050 port
#      otherwise , go step (1) again.


#stop
python kfslaunch.py -f machines.cfg  -S

#Checking
/usr/local/kfs-0.5/build/bin/tools/kfsping -m -s HT248 -p 20000
Up servers: 2
s=192.168.65.155, p=30000, rack=65, used=0(GB), free=43.8144(GB), util=0%, nblocks=0, lastheard=8 (sec), ncorrupt=0, nchunksToMove=0, numDrives=1
s=192.168.65.155, p=40000, rack=65, used=0(GB), free=43.8144(GB), util=0%, nblocks=0, lastheard=8 (sec), ncorrupt=0, nchunksToMove=0, numDrives=1

/usr/local/kfs-0.5/build/bin/tools/kfsping -c -s HT248 -p 20000
Meta-server:  0
Total-space: 0 (MB)
Used-space: 0 (MB)

/usr/local/kfs-0.5/build/bin/tools/kfsping -c -s HT248 -p 30000
Meta-server: kfs 20100
Total-space: 44866 (MB)
Used-space: 0 (MB)

/usr/local/kfs-0.5/build/bin/tools/kfsping -c -s HT248 -p 40000
Meta-server: kfs 20100
Total-space: 44866 (MB)
Used-space: 0 (MB)


elinks -dump http://HT248:20050/

                                    KFS Status

   Started at               : Tue Aug 21 11:32:37 2012
   Total space              : 400.00 GB
   Used space               : 0.00 bytes
   WORM mode                : Disabled
   Number of alive nodes    : 2
   Number of dead nodes     : 0
   Number of retiring nodes : 0

                                All Nodes
    Chunkserver  # of drives Used   Free   Used% # of blocks Last heard
   172.16.43.248      1      0(GB) 200(GB) 0.00            0   11 (sec)
   172.16.43.248      1      0(GB) 200(GB) 0.00            0   11 (sec)

cd /usr/local/kfs-0.5/build/bin/tools

[root@kfs tools]# ./cptokfs -s HT248 -p 20000 -d ~/install.log -k /
08-17-2012 12:44:19.994 DEBUG - (KfsClient.cc:636) Connecting to metaserver at: kfs:20000

[root@kfs tools]# ./cpfromkfs -s HT248 -p 20000 -d ~/sean.ply -k /install.log
08-17-2012 12:46:32.648 DEBUG - (KfsClient.cc:636) Connecting to metaserver at: kfs:20000
[root@kfs tools]# diff ~/sean.ply ~/install.log


[root@kfs tools]# dd if=/dev/urandom of=/root/random bs=65536 count=1024
1024+0 records in
1024+0 records out
67108864 bytes (67 MB) copied, 7.49119 s, 9.0 MB/s
[root@kfs tools]# ./cptokfs -s kfs -p 20000 -d ~/random -k /
08-17-2012 13:22:14.467 DEBUG - (KfsClient.cc:636) Connecting to metaserver at: kfs:20000
[root@kfs tools]# ./cpfromkfs -s kfs -p 20000 -d ~/random.back -k /random
08-17-2012 13:22:44.098 DEBUG - (KfsClient.cc:636) Connecting to metaserver at: kfs:20000
[root@kfs tools]# md5sum ~/random
9c9bc92c27e40ef7e54b6f858d3d2852  /root/random
[root@kfs tools]# md5sum ~/random.back
9c9bc92c27e40ef7e54b6f858d3d2852  /root/random.back


[root@kfs tools]# ./kfsshell -s HT248 -p 20000
KfsShell> ls -l
aabb/   Aug 16 17:23    (dir)
boost_1_40_0.tar.gz     Aug 16 18:07    4096
cshrc   Aug 16 16:43    100
db-4.8.26.tar.gz        Aug 16 16:37    4096
dumpster/       Aug 16 16:29    (dir)
install.log     Aug 17 12:44    15470
testkfs Aug 17 10:01    4096
user/   Aug 16 17:16    (dir)

或者

[root@kfs tools]# cd /root/code/kfs/scripts
[root@kfs scripts]# vi kfsshell.py
#    cmd = "%s/tools/KfsShell -s %s -p %d" % (bindir, node, port)
    cmd = "%s/tools/kfsshell -s %s -p %d" % (bindir, node, port)
[root@kfs scripts]# python kfsshell.py -f machines.cfg -b /root/code/kfs/build/bin
KfsShell>

------------------------------Mount Fuse----------------------------------------------------


vi /usr/local/kfs-0.5/build/bin/kfs.prp
metaServer.name=HT248
metaServer.port=20000

cd /usr/local/kfs-0.5/scripts
python kfslaunch.py -f machines.cfg  -s


[root@kfs scripts]# cd ../build/bin
[root@kfs bin]# mkdir /mnt/kfs
[root@kfs bin]# ./kfs_fuse /mnt/kfs -f

--------------------------KFS MakeFile of Examples------------------------------------------

AR=ar
ARFLAGS=-cr
RL=ranlib
RLFLAGS=
CC=g++
CXXFLAGS=-I. -Wall
CP=cp
LDFLAGS=
ifneq ($(RELEASE),yes)
CXXFLAGS += -g
LDFLAGS += -g
endif
CXXFLAGS += -I../src/cc/ -I/usr/include/ -I../src/cc/libkfsClient/
LDFLAGS += -L../build/lib/static/
EXTLIBS=-lkfsClient -lkfsCommon -lkfsIO -lqcdio -lpthread -lrt -lssl
DOWNLOAD_SRC= \ KfsSample_main.cc OBJ_DOWNLOAD=$(DOWNLOAD_SRC:.cpp=.o,.cc=.o)
TARGETS=testkfs
all: $(TARGETS)
testkfs: $(OBJ_DOWNLOAD)
    $(CC) $(LDFLAGS) $(CXXFLAGS) -o $@ $^ $(EXTLIBS)
%.o:%.cpp
    $(CC) $(CXXFLAGS) -c -o $@ $<
clean:
    rm -f *~ $(OBJS) $(TARGETS)

#testing
./testkfs  -s 192.168.65.155 -p 20000


[FAQ]

(1)boost version maybe cause problem(error) when hypertable making source code , such as
       hypertable struct std::basic_string<char, std::char_traits<char>, std::allocator<char> > ...
   see boost installation recommend above.
(2)libboost_regex.so may cause a error in 98% progression. because kosmosBroker could not find it.
   vi /usr/local/hypertable-x.x.x.x/cmake/FindKfs.cmake , and reference above see what you could do in it.
(3) rpm and yum
   1) find something in yum server, for example libcrypto.so.6
     yum provides \*/libcrypto.so.6
   2) find something in rpm that already installed
     rpm -ql vsftpd

     <if not installed , and you got the rpm file>
     rpm -qpl vsftpd.x.x.rpm

     <rpm infomation>
     rpm -qi(p) vsftpd(.x.x.rpm)

     <what rpm package do the shell command when installing>
     rpm -q(p) --scripts vsftpd(.x.x.rpm)

     <rpm version number>
     rpm -q(p) --qf '%{VERSION}' vsftpd(.x.x.rpm)
          
    

2012年8月10日 星期五

[KFS](KosmosFS) 安裝方法二

這次修改原本的安裝,多數以rpm packages安裝為主
(1)系統環境準備
    此次使用的環境,使用為virtual machine來實現
       host : CentOS-6.0
       kernel: 2.6.32-71.el6.x86_64 #1 SMP
       MemTotal:        7891424 kB(/proc/meminfo)
       CPU(Quad core)      : Intel(R) Core(TM) i3-2130 CPU @ 3.40GHz
       HDD: 1TB*2 (LVM ==> 2T)
       

       guest: CentOS-6.0
       kernel: 2.6.32-71.el6.x86_64 #1 SMP
       MemTotal:        1G
       CPU *1
       HDD: 100GB

       KFS的環境使用virtual machine的NAT環境,也把dhcp打開
       guest總共佈建4台,1台metaserver, 3台chunkserver
       這4台/etc/hosts設定如下
         192.168.65.1 mserver    #yum/ftp server
         192.168.65.150 t01nn    #metaserver
         192.168.65.151 t01d1    #chunkserver1
         192.168.65.152 t01d2    #chunkserver2
         192.168.65.153 t01d3    #chunkserver3
(2)系統安裝
使用kickstart檔來進行初期的部署(4台guest都照此安裝),所有rpm在這個階段都安裝完成,簡單描述幾個重點
        :
        selinux --disabled
        firewall --disabled
        :

        bootloader --location=mbr --driveorder=sda --append="rhgb quiet"
        :
        #在host,我們佈建了vsftpd以及httpd,
        #預設在/var/ftp/pub,/var/www/html(soft link to /var/ftp/pub)
        install
        url --url=http://192.168.65.1/pub
        :
        clearpart --drives=sda --all --initlabel
        part / --asprimary --fstype="ext4" --ondisk=sda --grow --size=1024
        part /boot --asprimary --fstype="ext4" --ondisk=sda --size=200
        part swap --fstype="swap" --ondisk=sda --recommended
        :


        #這裡將要安裝的套件一併在下面一一指定安裝
        %packages
        @core
        @server-policy
        ftp
        telnet
        ntpdate
        gcc
        gcc-c++
        make
        cmake
        log4cpp
        boost
        boost-devel
        expat
        openssl-devel
        java-1.6.0-openjdk
        java-1.6.0-openjdk-devel
        uuid
        uuid-devel
        libuuid
        libuuid-devel
        ant
        python
        python-devel

        fuse-devel
        fuse-libs
        %end

        %post
        #接下來的設定,可依個人喜好,把系統預設安裝的設定項,在這裡一併完成
        #筆者在這裡接連把下列的設定一併完成
        # 1.  /etc/sysconfig/network-scripts/ifcfg-eth0  設定網卡
        # 2.  /etc/sysconfig/network   設定hostname
        #      /etc/hosts
        #      /etc/resolv.conf
        # 3.  安裝xfsprogs相關套件,由於預設CentOS 6.0的安裝光碟沒有,於是下載後
        #      置入yum/ftp server上,在%post時補充安裝
        #      xfsprogs-3.1.1-7.el6.x86_64.rpm
        #      xfsprogs-devel-3.1.1-7.el6.x86_64.rpm

        # 4.  kfs-0.5.tar.gz的安裝 --> /usr/local/kfs-0.5
        #      詳細安裝可見[KFS](KosmosFS) 安裝方法一
        # 5.  /usr/local/kfs-0.5/scripts/machines.cfg
        # 6.  /usr/local/kfs-0.5/conf/machines.txt
        # 7.  /usr/local/kfs-0.5/conf/ChunkServer.prp
        # 8.  Fuse掛載KFS
        #      詳細安裝可見KFS mount by Fuse
        # 9.  設定方便的scripts, 可將kfs setup, start, stop及mount等 指令設置成scripts
        # 10. 更新套件,如果在這個步驟啟動KFS,fuse mount KFS時,
        #       可能會發生coredump。有可能的原因Centos 6.0是預設rpm套件太舊。
        #       筆者經過以下套件更新後,KFS mount即恢復正常(沒有一一試過是哪個太舊)
        #       如果不想輸入這麼長的一串更新套件,可以直接連上Centos 預設的yum server
        #       進行yum update即可
        #        rpm -Uvh ~/pkgs/boost-1.41.0-11.el6_1.2.x86_64.rpm  \
        #                       ~/pkgs/boost-date-time-1.41.0-11.el6_1.2.x86_64.rpm \
        #                       ~/pkgs/boost-devel-1.41.0-11.el6_1.2.x86_64.rpm  \
        #                       ~/pkgs/boost-filesystem-1.41.0-11.el6_1.2.x86_64.rpm \
        #                       ~/pkgs/boost-graph-1.41.0-11.el6_1.2.x86_64.rpm \
        #                       ~/pkgs/boost-iostreams-1.41.0-11.el6_1.2.x86_64.rpm \
        #                       ~/pkgs/boost-program-options-1.41.0-11.el6_1.2.x86_64.rpm \
        #                       ~/pkgs/boost-python-1.41.0-11.el6_1.2.x86_64.rpm \
        #                       ~/pkgs/boost-regex-1.41.0-11.el6_1.2.x86_64.rpm \
        #                       ~/pkgs/boost-serialization-1.41.0-11.el6_1.2.x86_64.rpm \
        #                       ~/pkgs/boost-signals-1.41.0-11.el6_1.2.x86_64.rpm \
        #                       ~/pkgs/boost-system-1.41.0-11.el6_1.2.x86_64.rpm \
        #                       ~/pkgs/boost-test-1.41.0-11.el6_1.2.x86_64.rpm \
        #                       ~/pkgs/boost-thread-1.41.0-11.el6_1.2.x86_64.rpm \
        #                       ~/pkgs/boost-wave-1.41.0-11.el6_1.2.x86_64.rpm \
        #                       ~/pkgs/cpp-4.4.6-4.el6.x86_64.rpm \
        #                       ~/pkgs/expat-2.0.1-11.el6_2.x86_64.rpm \
        #                       ~/pkgs/fuse-devel-2.8.3-4.el6.x86_64.rpm \
        #                       ~/pkgs/fuse-libs-2.8.3-4.el6.x86_64.rpm \
        #                       ~/pkgs/gcc-4.4.6-4.el6.x86_64.rpm \
        #                       ~/pkgs/gcc-c++-4.4.6-4.el6.x86_64.rpm \
        #                       ~/pkgs/libblkid-2.17.2-12.7.el6.x86_64.rpm \
        #                       ~/pkgs/libgcc-4.4.6-4.el6.x86_64.rpm \
        #                       ~/pkgs/libgomp-4.4.6-4.el6.x86_64.rpm \
        #                       ~/pkgs/libstdc++-4.4.6-4.el6.x86_64.rpm \
        #                       ~/pkgs/libstdc++-devel-4.4.6-4.el6.x86_64.rpm \
        #                       ~/pkgs/libuuid-2.17.2-12.7.el6.x86_64.rpm \
        #                       ~/pkgs/libuuid-devel-2.17.2-12.7.el6.x86_64.rpm \
        #                       ~/pkgs/make-3.81-20.el6.x86_64.rpm \
        #                       ~/pkgs/ntpdate-4.2.4p8-2.el6.centos.x86_64.rpm \
        #                       ~/pkgs/openssl-1.0.0-20.el6_2.5.x86_64.rpm \
        #                       ~/pkgs/openssl-devel-1.0.0-20.el6_2.5.x86_64.rpm \
        #                       ~/pkgs/python-2.6.6-29.el6_2.2.x86_64.rpm \
        #                       ~/pkgs/python-devel-2.6.6-29.el6_2.2.x86_64.rpm \
        #                       ~/pkgs/python-libs-2.6.6-29.el6_2.2.x86_64.rpm \
        #                       ~/pkgs/telnet-0.17-47.el6.x86_64.rpm \
        #                       ~/pkgs/util-linux-ng-2.17.2-12.7.el6.x86_64.rpm \
        #                       ~/pkgs/java-1.6.0-openjdk-1.6.0.0-1.48.1.11.3.el6_2.x86_64.rpm \
        #                       ~/pkgs/java-1.6.0-openjdk-devel-1.6.0.0-1.48.1.11.3.el6_2.x86_64.rpm

 (3)系統啟動
     系統啟動需要再執行以下作業
     1)ssh password-less
        利用這篇ssh password-less session config的做法 ,將t01nn, t01d1, t01d2, t01d3這幾台都設定為彼此不需要密碼登入
     2)在t01d1, t01d2, t01d3都執行以下指令
           rsync -av /usr/local/kfs-0.5/ t01d1:/usr/local/kfs-0.5/
           rsync -av /usr/local/kfs-0.5/ t01d2:/usr/local/kfs-0.5/
           rsync -av /usr/local/kfs-0.5/ t01d3:/usr/local/kfs-0.5/
    3)setup your KFS
         cd /usr/local/kfs-0.5/scripts
          python kfssetup.py -f machines.cfg -m ../conf/machines.txt -b ../build -w ../webui
    4) start up your KFS
          python /usr/local/kfs-0.5/scripts/kfslaunch.py -f /usr/local/kfs-0.5/scripts/machines.cfg -m /usr/local/kfs-0.5/conf/machines.txt -s

    5) mount your KFS
         /usr/local/kfs-0.5/build/bin/kfs_fuse /mnt/kfs -f -o allow_other
     
後記:
目前的安裝方法,己經縮短安裝時間, 約20分鐘即可在Virtual Machine將這4台KFS架設完成,但為求日後便利:
1)可以製作便利的ssh password-less scripts, 日後在主機異動時,可以快速設定
2)  可以製作便利的server-cnf-update scripts, 便利日後主機異動時,更新設定
     machines.txt machines.cfg ChunkServer.prp
     rsync...
3)自製kfs rpm,讓安裝更快速便利(目前kfs安裝約要5分鐘左右)且更適合大量部署
     

2012年8月4日 星期六

yum download cache & yum.conf

Sometimes, we always want to keep what you just download the packages (.rpm) via yum install ... command. Once you cloud change working directory to /var/cache/yum/.../... , and then all packages had gone after yum was completed.
/etc/yum.conf allow you to keep the cached packages and other parameter for more functions.

cachedir=/var/cache/yum    #cached packages destination
keepcache=0                #1 for keeping packages, 0 will remove the package after installed
metadata_expire=1800   # in seconds

Otherwise, you could man yum.conf for more information .

文章分類