xml config lookup shell scripts

例如可以用下面的scripts來查dfs.replication的設定值為何 [root@master02.dev ~]>cat ~/tttt.sh #!/bin/bash SEARCHPATH="./" if [ -z "$1" ]; then SEARCHPATH="./" else SEARCHPATH="$1" fi for file in `find $SEARCHPATH -name "*xml"` do echo $file cat $file | awk -F "\n" 'BEGIN{a=-1}{if(a < 0) { match($1 , pattern , ar); if( RSTART > 0) a=range; } if( a > 0) { a--; print markhead,$1; } }' range=2 pattern='dfs.replication' markhead='#' done [root@master02.dev process]>cd [root@master02.dev ~]>cd /var/run/cloudera-scm-agent/process [root@master02.dev process]>~/tttt.sh ./12-mapreduce-TASKTRACKER/mapred-site.xml ./12-mapreduce-TASKTRACKER/core-site.xml ./12-mapreduce-TASKTRACKER/hdfs-site.xml # <name>dfs.replication</name> # <value>3</value> ./13-mapreduce-JOBTRACKER/mapred-queue-acls.xml ./13-mapreduce-JOBTRACKER/fair-scheduler.xml ./13-mapreduce-JOBTRACKER/mapred-site.xml ./13-mapreduce-JOBTRACKER/core-site.xml ./13-mapreduce-JOBTRACKER/capacity-scheduler.xml ./13-mapreduce-JOBTRACKER/hdfs-site.xml # <name>dfs.replication</name> # <value>3</value> ./9-hdfs-NAMENODE/core-site.xml ./9-hdfs-NAMENODE/hdfs-site.xml # <name>dfs.replication</name> # <value>3</value> ./19-deploy-client-config/hadoop-conf/core-site.xml ./19-deploy-client-config/hadoop-conf/hdfs-site.xml # <name>dfs.replication</name> # <value>3</value> ./10-hdfs-DATANODE/core-site.xml ./10-hdfs-DATANODE/hdfs-site.xml # <name>dfs.replication</name> # <value>3</value> ./7-hdfs-NAMENODE-format/core-site.xml ./7-hdfs-NAMENODE-format/hdfs-site.xml # <name>dfs.replication</name> # <value>3</value> ./20-deploy-client-config/hadoop-conf/mapred-site.xml ./20-deploy-client-config/hadoop-conf/core-site.xml ./20-deploy-client-config/hadoop-conf/hdfs-site.xml # <name>dfs.replication</name> # <value>3</value> ./11-hdfs-NAMENODE-createtmp/core-site.xml ./11-hdfs-NAMENODE-createtmp/hdfs-site.xml # <name>dfs.replication</name> # <value>3</value> ./8-hdfs-SECONDARYNAMENODE/core-site.xml ./8-hdfs-SECONDARYNAMENODE/hdfs-site.xml # <name>dfs.replication</name> # <value>3</value>

Easy Minutely Candle-Stick Data By MapReduce under CDH4.2.0 / CM4.5.0--(1) JAVA

Environment:

OS: CentOS 6.2
Hadoop: CDH4.2.0 install via CM4.5.0
Hosts:
     Hostname  processes          ip
     ========  =================  ================
     master01  Namenode(CDH4)     192.168.122.11
     smaster   SecondaryNamenode  192.168.122.10
               JobTracker
     knode020  Datanode           192.168.122.20
               TaskTracker
     knode021  Datanode           192.168.122.21
               TaskTracker

MapReduce Version: 0.20.2

TWSE(Stock and warrant ticks)

  |

  +-> socket client(with libhdfs.so)

       |

       +-> HDFS(Stored by date)

             |

             +->MapReduce(Candle.java, candle.jar)

                  |

                  +-> HDFS(minutely candle stick data)

Map & Reduce Classes

   public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, Text> {

     private Text keytext = new Text();

     private Text valuetext = new Text();

     public void map(LongWritable key, Text value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException {

       int cnt = 0;

       String line = value.toString();

       String timestamp = new String();

       String symbol = new String();

       String SEQN = new String();

       String HiLoMark = new String();

       String MatchTime = new String();

       String MatchPrice = new String();

       String MatchVolumn = new String();

       StringTokenizer tokenizer = new StringTokenizer(line,"\t\n"); // default delim = " \t\n\r\f"

       while (tokenizer.hasMoreTokens() && cnt <7 ) {

         switch(cnt)

         {

         case 0:// timestamp

             timestamp = tokenizer.nextToken();

             break;

         case 1:// symbol

             symbol = tokenizer.nextToken();

             break;

         case 2:// SEQN

             SEQN = tokenizer.nextToken();

             break;

         case 3:// MatchTime

             MatchTime = tokenizer.nextToken();

             break;

         case 4:// HiLoMark

             HiLoMark = tokenizer.nextToken();

             break;

         case 5:// MatchPrice

             MatchPrice = tokenizer.nextToken();

             break;

         case 6:// MatchVolumn

             break;

         case 6:// MatchVolumn

             MatchVolumn = tokenizer.nextToken();

             break;

         default: // order book

             tokenizer.nextToken();

             break;

         }

         cnt++;

       }

       if(!MatchPrice.matches("-"))/*only match record*/

       {

         //keytext.set( String.format("%s\t%s", symbol,MatchTime.substring(0,5)));

         keytext.set( String.format("%s\t%s", symbol,MatchTime.substring(0,5)));

         valuetext.set( String.format("%s\t%s\t%s\t%s\t%s\t%s",MatchTime.substring(6,8),SEQN,timestamp,HiLoMark,MatchPrice,MatchVolumn));

         output.collect(keytext, valuetext);

       }

     }

   }

   public static class Reduce extends MapReduceBase implements Reducer<Text, Text, Text, Text> {

     private int high = 0;

     private int open = 0;

     private int low = 0;

     private int close = 0;

     private int vol = 0;

     private int open_seqn = 0;

     private int close_seqn = 0;

     private Text candletext = new Text();

     public void reduce(Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter) throws IOException {

       high = 0;

       open = 0;

       low = 0;

       close = 0;

       vol = 0;

       open_seqn = 0;

       close_seqn=0;

       while (values.hasNext()) {

         String Value=values.next().toString();

         StringTokenizer ValueTokenizer = new StringTokenizer(Value,"\t"); // default delim = " \t\n\r\f"

         int ii = 0;

         int seqn = 0;

         while (ValueTokenizer.hasMoreTokens() && ii <6 ) {

           int val = 0;

           String column = ValueTokenizer.nextToken();

           try

           {

             switch(ii++)

             {

             case 1: //SEQN

               seqn = Integer.parseInt(column);

               break;

             case 4:// MatchPrice

               val = Integer.parseInt(column);

               if(open_seqn == 0 || open_seqn  > seqn)

               {

                 open = val;

                 open_seqn = seqn;

               }

               if(high == 0 || val > high)

                 high = val;

               if(low == 0 || val < low)

                 low = val;

               if(close_seqn == 0 || close_seqn < seqn)

               {

                 close = val;

                 close_seqn = seqn;

               }

               break;

             case 5:// MatchValumn

               val = Integer.parseInt(column); /*total volumn*/

               if(val > vol)

                 vol=val;

               break;

             default:

               break;

             }

           }

           catch (NumberFormatException e)

           {

             System.err.println(e.toString());

             System.err.println("Integer.parseInt Error");

           }

           catch (Exception e)

           {

             System.err.println("Other Error"+e.getMessage());

           }

         }

       }

       candletext.set( String.format("%d\t%d\t%d\t%d\t%d",open,high,low,close,vol));

       output.collect(key, candletext);

     }

   }

                                                                                                       

Compile it:

[kccs@master01 candle]$ cat compile.sh

#!/bin/bash

if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then

        source /usr/lib/bigtop-utils/bigtop-detect-javahome

else

        source /usr/libexec/bigtop-detect-javahome

fi

export CLASSPATH=/etc/hadoop/conf

for file in `ls /usr/lib/hadoop/client/*.jar`

do

  export CLASSPATH=$CLASSPATH:$file

done

export CLASSPATH=$CLASSPATH:/usr/lib/hadoop/hadoop-annotations.jar

export LD_LIBRARY_PATH="$JAVA_HOME/jre/lib/amd64/server/"

if [ ! -e MyJava ]; then

mkdir MyJava

fi

javac -d MyJava Candle.java

jar -cvf candle.jar -C MyJava .

if [ ! -e ~/bin/candle_stick ]; then

mkdir ~/bin/candle_stick

fi

Run it:

[kccs@master01 candle]$ cat run2.sh

#!/bin/bash

WHO=`whoami`

inputPath=/kccs/$1/

outputPath=/user/$WHO/candle/$1

hdfs dfs -test -e $inputPath

if [ ! $? == "0" ]; then

echo "no tick data exist !!!"

exit 1

fi

WHO=`whoami`

hdfs dfs -test -e $outputPath

if [ $? == "0" ]; then

hdfs dfs -rm -r $outputPath

fi

echo "candle stick start...."

hadoop jar /home/kccs/src/ex_mapreduce/candle/candle.jar Candle $inputPath $outputPath

Result:

[kccs@master01 candle]$ hdfs dfs -cat /user/kccs/candle/20130730/part-00000

:

:

:

9958    12:24   1275    1280    1275    1280    543

9958    12:26   1280    1285    1280    1285    570

9958    12:27   1285    1285    1285    1285    575

9958    12:28   1285    1285    1285    1285    577

9958    12:30   1285    1285    1280    1280    580

9958    12:31   1280    1280    1280    1280    582

9958    12:32   1280    1280    1280    1280    587

9958    12:33   1280    1280    1280    1280    592

9958    12:34   1280    1280    1280    1280    612

9958    12:38   1275    1275    1275    1275    614

9958    12:43   1275    1275    1275    1275    621

9958    12:44   1275    1275    1275    1275    631

9958    12:45   1280    1280    1280    1280    632

9958    12:47   1275    1275    1275    1275    634

9958    12:51   1275    1275    1275    1275    644

9958    12:53   1275    1275    1275    1275    645

9958    12:58   1275    1275    1275    1275    647

9958    13:00   1275    1275    1275    1275    649

9958    13:01   1275    1275    1275    1275    652

9958    13:06   1280    1280    1280    1280    653

9958    13:10   1280    1280    1280    1280    673

9958    13:11   1280    1280    1280    1280    675

9958    13:12   1280    1280    1280    1280    680

9958    13:18   1275    1275    1275    1275    710

9958    13:19   1275    1275    1275    1275    717

9958    13:22   1275    1275    1275    1275    718

9958    13:23   1280    1280    1280    1280    723

9958    13:24   1280    1280    1280    1280    724

9958    13:30   1275    1275    1275    1275    737

                                                                                                       

upgrading hypertable under cdh4

(0) environmental
     Hostname  processes          ip
     ========  =================  ================
     master01  Hypertable.Master  192.168.122.11
               Hyperspace
               DfsBroker
               RangeServer
               Namenode(CDH4)      
     smaster   SecondaryNamenode  192.168.122.10
               JobTracker
     knode020  DfsBroker          192.168.122.20
               RangeServer
               Datanode
               TaskTracker

     knode021  DfsBroker          192.168.122.21
               RangeServer
               Datanode
               TaskTracker

     old Hypertable Version: 

              hypertable-0.9.6.5.59309c4-linux-x86_64.tar.gz

     new Hypertable Version: 

              hypertable-0.9.7.7-linux-x86_64.rpm 

(1) download 0.9.7.7 rpm file
   ssh to master01 as root user:
   [root@master01~] cd /mnt/nfssvr/extras/hypertable/rpm
   [root@master01 rpm]# wget http://cdn.hypertable.com/packages/0.9.7.7/hypertable-0.9.7.7-linux-x86_64.rpm

(2) upgrading
     (a) stop your hypertable cluster(my Capfile is in /root)
     [root@master01 rpm] cd ~
     [root@master01 rpm] cap stop
     [root@master01 rpm] ps -ef | grep hyper
   # if there was any process exist, stop it

     [root@master01 rpm] /opt/hypertable/current/bin/stop-servers.sh

     (b) modify your Capfile
     [root@master01 rpm] cd ~
     [root@master01 ~] vi Capfile
         :

         #set :hypertable_version, "0.9.6.5.59309c4"
         set :hypertable_version, "0.9.7.7"
         #set :default_pkg, "/mnt/nfssvr/extras/hypertable/rpm/hypertable-0.9.6.5.59309c4-linux-x86_64.tar.gz"
         set :default_pkg, "/mnt/nfssvr/extras/hypertable/rpm/hypertable-0.9.7.7-linux-x86_64.rpm"

         :

         :

     (c) install it & upgrade it

     [root@master01 ~] cd ~

     [root@master01 ~] cap install_package

     [root@master01 ~] cap dist

     [root@master01 ~] ls /opt/hypertable

drwxr-xr-x 7 root root      4096 Jun  7 16:55 0.9.6.5.59309c4
drwxr-xr-x 8 root root      4096 Jun 10 14:05 0.9.7.7
lrwxrwxrwx 1 root root        15 Jun  7 16:55 current -> 0.9.6.5.59309c4
-rwxr-xr-x 1 root root 103614419 Jun  7 16:55 hypertable-0.9.6.5.59309c4-linux-x86_64.tar.gz

   #so far , /opt/hypertabl/0.9.7.7 was installed 

   #,but soft link "current" was still pointed to 0.9.6.5.59309c4

   #cap dist to update new version

     [root@master01 ~] cap upgrade

   #"cap upgrade" whill :

   # 1.verifying the format is Compatible (between old hypertable and new one). Also you could verify compatible by yourself "/opt/hypertable/current/upgrade-ok.sh"

   # 2.Copy folders(run/ conf/ run/...etc)from old(0.9.6.5.59309c4) version to new one

   # 3.redirect the soft link to "0.9.7.7"

     [root@master01 ~] ls /opt/hypertable

drwxr-xr-x 7 root root      4096 Jun  7 16:55 0.9.6.5.59309c4

drwxr-xr-x 7 root root      4096 Jun 10 14:06 0.9.7.7

lrwxrwxrwx 1 root root         7 Jun 10 14:06 current -> 0.9.7.7

-rwxr-xr-x 1 root root 103614419 Jun  7 16:55 hypertable-0.9.6.5.59309c4-linux-x86_64.tar.gz

      (d)soft link

      #disable  softlink liblog4cpp

     [root@master01 ~] vi /mnt/nfssvr/extras/lib_alias.sh

      :

      :

  #if [ ! -e /opt/hypertable/current/lib/liblog4cpp.so ]; then
  #    log4cpp_fname="$(ls /opt/hypertable/current/lib/liblog4cpp* | grep liblog4cpp.so.)"
  #    ln -s $log4cpp_fname       /opt/hypertable/current/lib/liblog4cpp.so

  #fi
  exit 0

    

     #relink lib alias

     [root@master01 ~] cap lib_alias

     [root@master01 ~]# ls -l /opt/hypertable/current/lib/*.so

           :

     lrwxrwxrwx 1 root root       45 2013-06-13 13:48 /opt/hypertable/current/lib/libevent.so -> /opt/hypertable/current/lib/libevent-1.4.so.2

           :

           :

           :

     lrwxrwxrwx 1 root root       46 2013-06-13 13:38 /opt/hypertable/current/lib/libthrift.so -> /opt/hypertable/current/lib/libthrift-0.8.0.so

     (e) start hypertable

     #Now you could start up hypertable normally. Under CDH4 , you need to do more before you startup (otherwise you would get a exception)

     

   [root@master01 ~]# cap shell
   cap> /opt/hypertable/current/bin/set-hadoop-distro.sh cdh4
    [establishing connection(s) to master01, knode020, knode021, smaster]
     ** [out :: knode020] Hypertable successfully configured for Hadoop cdh4
     ** [out :: knode021] Hypertable successfully configured for Hadoop cdh4
     ** [out :: master01] Hypertable successfully configured for Hadoop cdh4
     ** [out :: smaster] Hypertable successfully configured for Hadoop cdh4
    cap> quit
    exiting

   [root@master01 ~]# cap start

     (f) for makefile

   CC=g++
   CFLAGS=-c -Wall -D_REENTRANT -I/opt/hypertable/current/include \
          -I/opt/hypertable/current/include/thrift
   LDFLAGS=-rdynamic -L/opt/hypertable/current/lib -lHyperThrift \
           -lHyperCommon -lHypertable -lthrift -levent
   #        -lHyperCommon -lHypertable -lthrift -levent -llog4cpp
   BIN=../bin

   all: client_test candle candle2

   client_test: client_test.o
           $(CC) client_test.o $(LDFLAGS) -o client_test
           cp client_test $(BIN)

           :

           :

   client_test.o: client_test.cc
           $(CC) $(CFLAGS) client_test.cc

           :

           :

   clean:
           rm -rf *o client_test

           :

           :

see more:

     http://hypertable.com/documentation/administrator_guide/upgrading

    https://groups.google.com/forum/?fromgroups#!topic/hypertable-user/iuduinCcIjU