host : CentOS-6.0
kernel: 2.6.32-71.el6.x86_64 #1 SMP
MemTotal: 7891424 kB(/proc/meminfo)
CPU(Quad core) : Intel(R) Core(TM) i3-2130 CPU @ 3.40GHz
HDD： 1TB＊2 （LVM ＝＝＞ 2T）
guest: CentOS-6.0
kernel: 2.6.32-71.el6.x86_64 #1 SMP
MemTotal: 1G
CPU *1
HDD： 100GB
yum install gcc gcc-c++ make ntpdate cmake java-1.6.0-openjdk java-1.6.0-openjdk-devel
yum install icu libicu libicu-devel #for libboost_regex.so
rpm -Uvh http://ftp.sjtu.edu.cn/sites/download.fedora.redhat.com/pub/epel/6/x86_64/epel-release-6-7.noarch.rpm
yum install log4cpp log4cpp-devel expat expat-devel uuid uuid-devel libuuid libuuid-devel python python-devel fuse fuse-libs fuse-devel openssl openssl-devel ant zlib-devel
rpm -ivh xfsprogs-3.1.1-7.el6.x86_64.rpm xfsprogs-devel-3.1.1-7.el6.x86_64.rpm
yum install sigar sigar-devel bzip2 bzip2-devel rrdtool libevent-devel cronolog libedit-devel readline readline-devel
#if you need jemalloc jemalloc-devel , then install it via yum instal...
/usr/sbin/ntpdate -b time.stdtime.gov.tw;/sbin/hwclock --systohc --utc
cd /usr/local
wget http://hi.baidu.com/cpuramdisk/item/3410d0ccc53e3f0b0ad93a00
#or
#wget http://download.oracle.com/berkeley-db/db-4.8.26.tar.gz
tar -zxvf db-4.8.26.tar.gz
cd db-4.7.25/build_unix/
../dist/configure --enable-cxx (--enable-java;)(root permission is neccessary)
make
make install
echo “/usr/local/BerkeleyDB.4.8/lib” >> /etc/ld.so.conf.d /BerkeleyDB.conf
sudo /sbin/ldconfig
#I had been installed for boost-1.41.x via yum, but hypertable may not be built successful, an error
#hypertable struct std::basic_string<char, std::char_traits<char>, std::allocator<char> >
#when you re-install boost_1_49_0 as below, you may need to re-build kfs too.
#I recommend you remove /usr/local/kfs-0.5/build , and re-build it . Otherwise, you will get errors.
#That errors will be something link "libboost_regex-mt.so ..."
cd /usr/local
wget http://nchc.dl.sourceforge.net/project/boost/boost/1.49.0/boost_1_49_0.tar.gz
tar -zxvf boost_1_49_0.tar.gz
cd boost_1_49_0/tools/build/v2/
./bootstrap.sh
./b2 install --prefix=/usr/local
cd /usr/local/boost_1_49_0/
./bootstrap.sh --with-icu --with-libraries=regex,filesystem,chrono,date_time,exception,iostreams,locale,program_options,python,random,serialization,signals,system,test,thread,timer,wave
./b2
./b2 install
ln -s /usr/local/boost_1_49_0/stage/lib/libboost_regex.a /usr/lib64/libboost_regex.a
ln -s /usr/local/boost_1_49_0/stage/lib/libboost_regex.so /usr/lib64/libboost_regex.so
ln -s /usr/local/boost_1_49_0/stage/lib/libboost_regex.so.1.49.0 /usr/lib64/libboost_regex.so.1.49.0
cd /usr/local
wget http://ftp.heanet.ie/mirrors/sourceforge/k/project/ko/kosmosfs/kosmosfs/kfs-0.5/kfs-0.5.tar.gz
tar -zxvf kfs-0.5.tar.gz
mkdir -p kfs-0.5/build
cd kfs-0.5/build
vi ../CMakeLists.txt
#line 110(after 'set (CMAKE_BUILD_TYPE "Debug")' ) : add 2 lines java setting
set (JAVA_INCLUDE_PATH "/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64/include")
set (JAVA_INCLUDE_PATH2 "/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64/include")
set (JAVA_INCLUDE_PATH2 "/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64/include")
#line 137: fill fuse setting
SET(Fuse_LIBRARY_DIR "/lib64")
SET(Fuse_INCLUDE_DIR "/usr/include")
cmake -D CMAKE_BUILD_TYPE=RelWithDebInfo /usr/local/kfs-0.5
gmake
gmake install
cd ..
ant jar
vi ~/.bashrc
export CLASSPATH=${CLASSPATH}:/usr/local/kfs-0.5/build/kfs-0.5.jar
cd /usr/local/kfs-0.5/src/cc/access
vi kfs_setup.py
kfsext = Extension('kfs',
include_dirs = ['/usr/local/kfs-0.5/src/cc/', '/usr/local/include/boost/'],
## use full path name in it.
python kfs_setup.py /usr/local/kfs-0.5/build/lib/ build
python kfs_setup.py /usr/local/kfs-0.5/build/lib/ install
python kfs_setup.py /usr/local/kfs-0.5/build/lib/ install --home=/usr/local/kfs-0.5/build/lib
vi ~/.bashrc
export PYTHONPATH=${PYTHONPATH}:/usr/local/kfs-0.5/build/lib/lib64/python
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/kfs-0.5/build/lib
wget http://mirror.yandex.ru/mandriva/devel/cooker/SRPMS/contrib/release/re2-0-1.src.rpm
rpm -ivh re2-0-1.src.rpm
cd ~/rpmbuild/SOURCES/
tar -zxvf re2.tar.gz
cd re2
make;make install
cd /usr/local
wget http://snappy.googlecode.com/files/snappy-1.0.4.tar.gz
tar -zxvf snappy-1.0.4.tar.gz
cd snappy-1.0.4
./configure; make; make install
#<optional> if you don't need ceph file system, skip this step
cd /usr/local
wget http://ceph.newdream.net/download/ceph-0.22.2.tar.gz
tar xzvf ceph-0.49.tar.gz
cd ceph-0.49
./configure; make; make install
cd /usr/local
wget http://archive.apache.org/dist/thrift/0.8.0/thrift-0.8.0.tar.gz
tar -zxvf thrift-0.8.0.tar.gz
cd thrift-0.8.0
./configure
make
make install
/sbin/ldconfig
wget http://magelan.googlecode.com/files/hyperic-sigar-1.6.4.zip
unzip hyperic-sigar-1.6.4.zip
cp hyperic-sigar-1.6.4/sigar-bin/include/*.h /usr/local/include
cp hyperic-sigar-1.6.4/sigar-bin/lib/libsigar-amd64-linux.so /usr/local/lib
echo "/usr/local/lib" >> /etc/ld.so.conf.d/normal.conf
/sbin/ldconfig
cd /usr/local
http://hypertable.googlecode.com/files/hypertable-0.9.6.1-src.tar.gz
tar -zxvf hypertable-0.9.6.1-src.tar.gz
cd hypertable-0.9.6.1
#if you don't wanna kosmosBroker, skip this step
vi CMakeLists.txt
#update "find_package(Kfs)" into
find_package(Kfs REQUIRED)
vi cmake/FindKfs.cmake
#Add a line about Line 33: (after "find_path(Kfs_INCLUDE_DIR kfs/KfsClient.h "...)
/usr/local/kfs-0.5/build/include
#Update Line 42: (after "find_library(${lib}_LIB NAMES ${lib}")
#before
find_library(${lib}_LIB NAMES ${lib}
PATHS /opt/kfs/lib/static /opt/kfs/lib /opt/local/lib /usr/local/lib
#after
find_library(${lib}_LIB NAMES ${lib}
PATHS /opt/kfs/lib/static /opt/kfs/lib /opt/local/lib /usr/local/lib /usr/local/kfs-0.5/build/lib
#Add a line about Line 54: (after "find_library(Crypto_LIB "...)
find_library(regex_LIB NAMES boost_regex PATHS /usr/local/lib)
#Update Line 59: (after "if (Kfs_INCLUDE_DIR AND kfsClient_LIB)...")
#before
if (Kfs_INCLUDE_DIR AND kfsClient_LIB)
set(Kfs_FOUND TRUE)
set( Kfs_LIBRARIES ${kfsClient_LIB} ${kfsIO_LIB} ${kfsCommon_LIB}
${qcdio_LIB} ${Crypto_LIB})
#after
if (Kfs_INCLUDE_DIR AND kfsClient_LIB)
set(Kfs_FOUND TRUE)
set( Kfs_LIBRARIES ${kfsClient_LIB} ${kfsIO_LIB} ${kfsCommon_LIB}
${qcdio_LIB} ${Crypto_LIB} ${regex_LIB})
mkdir build
cd build
cmake ../
gmake
gmake install
#all bin file default will copy to /opt/hypertable/.
--------------------------KFS Deployment----------------------------------------------------
ssh-keygen -t dsa
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
cd /usr/local/kfs-0.5/scripts
vi ../webui/all-machines.txt
HT248
vi machines.cfg
# KFS Machine configuration file
[metaserver]
node: HT248
clusterkey: kfs
rundir: /home/kfs/meta
baseport: 20000
[chunkserver1]
node: HT248
rundir: /home/kfs/chunk1
baseport: 30000
space: 200 G
[chunkserver2]
node: HT248
rundir: /home/kfs/chunk2
baseport: 40000
space: 200 G
#single node
python kfssetup.py -f machines.cfg -b ../build -w ../webui -s
#multiple nodes
python kfssetup.py -f machines.cfg -b ../build -w ../webui
#startup
python kfslaunch.py -f machines.cfg -s
# after startup kfs , if webui didn't listen 20050 port . you should :
# (1) stop kfs
# (2) remove /home/kfs directory
# (3) re-setup kfs (python kfssetup.py -f machines.cfg ...)
# (4) check server.conf and all-machines.txt are exist at /home/kfs/meta/webui/
# (5) if webui config is ok , startup kfs . you could elinks to 20050 port
# otherwise , go step (1) again.
#stop
python kfslaunch.py -f machines.cfg -S
#Checking
/usr/local/kfs-0.5/build/bin/tools/kfsping -m -s HT248 -p 20000
Up servers: 2
s=192.168.65.155, p=30000, rack=65, used=0(GB), free=43.8144(GB), util=0%, nblocks=0, lastheard=8 (sec), ncorrupt=0, nchunksToMove=0, numDrives=1
s=192.168.65.155, p=40000, rack=65, used=0(GB), free=43.8144(GB), util=0%, nblocks=0, lastheard=8 (sec), ncorrupt=0, nchunksToMove=0, numDrives=1
/usr/local/kfs-0.5/build/bin/tools/kfsping -c -s HT248 -p 20000
Meta-server: 0
Total-space: 0 (MB)
Used-space: 0 (MB)
/usr/local/kfs-0.5/build/bin/tools/kfsping -c -s HT248 -p 30000
Meta-server: kfs 20100
Total-space: 44866 (MB)
Used-space: 0 (MB)
/usr/local/kfs-0.5/build/bin/tools/kfsping -c -s HT248 -p 40000
Meta-server: kfs 20100
Total-space: 44866 (MB)
Used-space: 0 (MB)
elinks -dump http://HT248:20050/
KFS Status
Started at : Tue Aug 21 11:32:37 2012
Total space : 400.00 GB
Used space : 0.00 bytes
WORM mode : Disabled
Number of alive nodes : 2
Number of dead nodes : 0
Number of retiring nodes : 0
All Nodes
Chunkserver # of drives Used Free Used% # of blocks Last heard
172.16.43.248 1 0(GB) 200(GB) 0.00 0 11 (sec)
172.16.43.248 1 0(GB) 200(GB) 0.00 0 11 (sec)
cd /usr/local/kfs-0.5/build/bin/tools
[root@kfs tools]# ./cptokfs -s HT248 -p 20000 -d ~/install.log -k /
08-17-2012 12:44:19.994 DEBUG - (KfsClient.cc:636) Connecting to metaserver at: kfs:20000
[root@kfs tools]# ./cpfromkfs -s HT248 -p 20000 -d ~/sean.ply -k /install.log
08-17-2012 12:46:32.648 DEBUG - (KfsClient.cc:636) Connecting to metaserver at: kfs:20000
[root@kfs tools]# diff ~/sean.ply ~/install.log
[root@kfs tools]# dd if=/dev/urandom of=/root/random bs=65536 count=1024
1024+0 records in
1024+0 records out
67108864 bytes (67 MB) copied, 7.49119 s, 9.0 MB/s
[root@kfs tools]# ./cptokfs -s kfs -p 20000 -d ~/random -k /
08-17-2012 13:22:14.467 DEBUG - (KfsClient.cc:636) Connecting to metaserver at: kfs:20000
[root@kfs tools]# ./cpfromkfs -s kfs -p 20000 -d ~/random.back -k /random
08-17-2012 13:22:44.098 DEBUG - (KfsClient.cc:636) Connecting to metaserver at: kfs:20000
[root@kfs tools]# md5sum ~/random
9c9bc92c27e40ef7e54b6f858d3d2852 /root/random
[root@kfs tools]# md5sum ~/random.back
9c9bc92c27e40ef7e54b6f858d3d2852 /root/random.back
[root@kfs tools]# ./kfsshell -s HT248 -p 20000
KfsShell> ls -l
aabb/ Aug 16 17:23 (dir)
boost_1_40_0.tar.gz Aug 16 18:07 4096
cshrc Aug 16 16:43 100
db-4.8.26.tar.gz Aug 16 16:37 4096
dumpster/ Aug 16 16:29 (dir)
install.log Aug 17 12:44 15470
testkfs Aug 17 10:01 4096
user/ Aug 16 17:16 (dir)
或者
[root@kfs tools]# cd /root/code/kfs/scripts
[root@kfs scripts]# vi kfsshell.py
# cmd = "%s/tools/KfsShell -s %s -p %d" % (bindir, node, port)
cmd = "%s/tools/kfsshell -s %s -p %d" % (bindir, node, port)
[root@kfs scripts]# python kfsshell.py -f machines.cfg -b /root/code/kfs/build/bin
KfsShell>
------------------------------Mount Fuse----------------------------------------------------
vi /usr/local/kfs-0.5/build/bin/kfs.prp
metaServer.name=HT248
metaServer.port=20000
cd /usr/local/kfs-0.5/scripts
python kfslaunch.py -f machines.cfg -s
[root@kfs scripts]# cd ../build/bin
[root@kfs bin]# mkdir /mnt/kfs
[root@kfs bin]# ./kfs_fuse /mnt/kfs -f
--------------------------KFS MakeFile of Examples------------------------------------------
AR=arARFLAGS=-cr
RL=ranlib
RLFLAGS=
CC=g++
CXXFLAGS=-I. -Wall
CP=cp
LDFLAGS=
ifneq ($(RELEASE),yes)
CXXFLAGS += -g
LDFLAGS += -g
endif
CXXFLAGS += -I../src/cc/ -I/usr/include/ -I../src/cc/libkfsClient/
LDFLAGS += -L../build/lib/static/
EXTLIBS=-lkfsClient -lkfsCommon -lkfsIO -lqcdio -lpthread -lrt -lssl
DOWNLOAD_SRC= \ KfsSample_main.cc OBJ_DOWNLOAD=$(DOWNLOAD_SRC:.cpp=.o,.cc=.o)
TARGETS=testkfs
all: $(TARGETS)
testkfs: $(OBJ_DOWNLOAD)
$(CC) $(LDFLAGS) $(CXXFLAGS) -o $@ $^ $(EXTLIBS)
%.o:%.cpp
$(CC) $(CXXFLAGS) -c -o $@ $<
clean:
rm -f *~ $(OBJS) $(TARGETS)
#testing
./testkfs -s 192.168.65.155 -p 20000
[FAQ]
(1)boost version maybe cause problem(error) when hypertable making source code , such ashypertable struct std::basic_string<char, std::char_traits<char>, std::allocator<char> > ...
see boost installation recommend above.
(2)libboost_regex.so may cause a error in 98% progression. because kosmosBroker could not find it.
vi /usr/local/hypertable-x.x.x.x/cmake/FindKfs.cmake , and reference above see what you could do in it.
(3) rpm and yum
1) find something in yum server, for example libcrypto.so.6
yum provides \*/libcrypto.so.6
2) find something in rpm that already installed
rpm -ql vsftpd
<if not installed , and you got the rpm file>
rpm -qpl vsftpd.x.x.rpm
<rpm infomation>
rpm -qi(p) vsftpd(.x.x.rpm)
<what rpm package do the shell command when installing>
rpm -q(p) --scripts vsftpd(.x.x.rpm)
<rpm version number>
rpm -q(p) --qf '%{VERSION}' vsftpd(.x.x.rpm)
