Commit d3368987 authored by Jan Siersch's avatar Jan Siersch
Browse files

added chukwa agent installation to computenode

parent 9e342dcb
if (( $# != 1 )); then
echo "usage: $0 [install|start|stop|restart]";
exit 1;
fi
doInstall=false;
doStatus=false;
doStop=false;
doStart=false;
if [ "$1" == "install" ]; then
doInstall=true;
elif [ "$1" == "status" ]; then
doStatus=true;
elif [ "$1" == "start" ]; then
doStart=true;
elif [ "$1" == "stop" ]; then
doStop=true;
elif [ "$1" == "restart" ]; then
doStart=true;
doStop=true;
else
echo "usage: $0 [install|start|stop|restart]";
exit 1;
fi
if $doInstall; then
yum -y install virt-top
groupadd chukwa
useradd -g chukwa chukwa
mkdir -p /home/chukwa
cp -r ./chukwa /home
chown -R chukwa:chukwa /home/chukwa
chmod -R 755 /home/chukwa
fi
if $doStatus; then
pidfile="/home/chukwa/pids/Agent.pid"
if [[ -e "$pidfile" ]]; then
echo "chukwa agent pid: $(cat $pidfile)"
else
echo "chukwa agent not running"
fi
fi
if $doStop; then
cd /home/chukwa
sudo -u chukwa /home/chukwa/bin/chukwa --config /home/chukwa/etc/chukwa agent stop
fi
if $doStart; then
cd /home/chukwa
sudo -u chukwa /home/chukwa/bin/chukwa --config /home/chukwa/etc/chukwa agent
fi
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# The Chukwa command script
#
# Environment Variables
#
# JAVA_HOME The java implementation to use. Overrides JAVA_HOME.
# CHUKWA_CONF_DIR Alternate conf dir. Default is ${CHUKWA_HOME}/conf.
#
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
. "$bin"/../libexec/chukwa-config.sh
# if no args specified, show usage
if [ $# = 0 ]; then
echo "Usage: chukwa [--config confdir] COMMAND"
echo "where COMMAND is one of:"
echo " agent run a Chukwa Agent"
echo " archive run the Archive Manager"
echo " collector run a Chukwa Collector"
echo " demux run the Demux Manager"
echo " dp run the Post Demux data processors"
echo " hicc run a HICC Webserver"
echo " droll run a daily rolling job (deprecated)"
echo " hroll run a hourly rolling job (deprecated)"
# Daily rolling and hourly rolling will be deprecated by retention processor
# echo " retention run the Retention Processor"
echo " version print the version"
echo "Utilities:"
echo " backfill run a back fill data loader utility"
echo " dumpArchive view an archive file"
echo " dumpRecord view a record file"
echo " tail start tailing a file"
echo "Most command print help when invoked w/o parameters."
exit 1
fi
# get arguments
COMMAND=$1
shift
if [ -f "${CHUKWA_CONF_DIR}/chukwa-env.sh" ]; then
. "${CHUKWA_CONF_DIR}/chukwa-env.sh"
fi
# Java parameters
if [ "$JAVA_HOME" != "" ]; then
JAVA_HOME=$JAVA_HOME
fi
if [ "$JAVA_HOME" = "" ]; then
echo "Error: JAVA_HOME is not set."
exit 1
fi
if [ "$HADOOP_CONF_DIR" != "" ]; then
CLASSPATH=${HADOOP_CONF_DIR}:${CLASSPATH}
fi
if [ "$CHUKWA_HICC_PORT" != "" ]; then
CHUKWA_HICC_PORT=$CHUKWA_HICC_PORT
fi
if [ "$CHUKWA_HICC_PORT" = "" ]; then
CHUKWA_HICC_PORT=4080
fi
BACKGROUND="true"
# configure command parameters
if [ "$COMMAND" = "agent" ]; then
APP='agent'
CLASS='org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent'
PID="Agent"
elif [ "$COMMAND" = "archive" ]; then
APP='archive'
CLASS='org.apache.hadoop.chukwa.extraction.archive.ChukwaArchiveManager'
PID='ArchiveManager'
elif [ "$COMMAND" = "collector" ]; then
APP='collector'
CLASS='org.apache.hadoop.chukwa.datacollection.collector.CollectorStub'
PID="Collector"
elif [ "$COMMAND" = "dp" ]; then
APP='postprocess'
CLASS='org.apache.hadoop.chukwa.extraction.demux.PostProcessorManager'
PID='PostProcessorManager'
elif [ "$COMMAND" = "demux" ]; then
APP='Demux'
CLASS='org.apache.hadoop.chukwa.extraction.demux.DemuxManager'
PID='DemuxManager'
elif [ "$COMMAND" = "hicc" ]; then
WEB_SERVICE_COMMON=`ls ${CHUKWA_HOME}/share/chukwa/webapps/hicc.war`
APP='hicc'
CLASS='org.apache.hadoop.chukwa.hicc.HiccWebServer'
CLASSPATH=${CLASSPATH}:${WEB_SERVICE_COMMON}:${HICC_JAR}
PID="hicc"
HOST=`hostname`
JAVA_OPT="${JAVA_OPT} -Djetty.host=${HOST} -Djetty.port=${CHUKWA_HICC_PORT} -Djava.net.preferIPv4Stack=true"
elif [ "$COMMAND" = "hroll" ]; then
APP='hroll'
CLASS='org.apache.hadoop.chukwa.extraction.demux.HourlyChukwaRecordRolling'
OPTS='rollInSequence true deleteRawdata true'
PID='HourlyChukwaRecordRolling'
elif [ "$COMMAND" = "droll" ]; then
APP='droll'
CLASS='org.apache.hadoop.chukwa.extraction.demux.DailyChukwaRecordRolling'
OPTS='rollInSequence true deleteRawdata true'
PID='DailyChukwaRecordRolling'
elif [ "$COMMAND" = "version" ]; then
echo `cat ${CHUKWA_HOME}/bin/VERSION`
exit 0
elif [ "$COMMAND" = "backfill" ]; then
APP='backfill'
CLASS='org.apache.hadoop.chukwa.tools.backfilling.BackfillingLoader'
elif [ "$COMMAND" = "dumpArchive" ]; then
APP='DumpArchive'
CLASS='org.apache.hadoop.chukwa.util.DumpArchive'
BACKGROUND="false"
elif [ "$COMMAND" = "dumpRecord" ]; then
APP='DumpRecord'
CLASS='org.apache.hadoop.chukwa.util.DumpRecord'
BACKGROUND="false"
elif [ "$COMMAND" = "dumpChunk" ]; then
APP='DumpChunk'
CLASS='org.apache.hadoop.chukwa.util.DumpChunks'
BACKGROUND="false"
elif [ "$COMMAND" = "tail" ]; then
if [ $# = 0 -o $# -gt 2 ]; then
echo "usage: startWatchingFile <filename> [datatype]"
exit 1
fi
filename=$1
if [ $# -ge 2 ]; then
datatype=$2
else
datatype=raw
fi
echo "tailing $datatype: $filename"
echo "add org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8 $datatype 0 $filename 0" | nc localhost 9093
exit 0
fi
if [ "$1" = "stop" ]; then
kill -TERM `cat ${CHUKWA_PID_DIR}/$PID.pid`
else
# run command
if [ "$BACKGROUND" = "false" ]; then
${JAVA_HOME}/bin/java ${JAVA_OPT} -Djava.library.path=${JAVA_LIBRARY_PATH} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DCHUKWA_DATA_DIR=${CHUKWA_DATA_DIR} -DAPP=${APP} -Dlog4j.configuration=chukwa-log4j.properties -classpath ${CHUKWA_CONF_DIR}:${CLASSPATH}:${CHUKWA_CLASSPATH}:${tools} ${CLASS} $OPTS $@
else
exec ${JAVA_HOME}/bin/java ${JAVA_OPT} -Djava.library.path=${JAVA_LIBRARY_PATH} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DCHUKWA_DATA_DIR=${CHUKWA_DATA_DIR} -DAPP=${APP} -Dlog4j.configuration=chukwa-log4j.properties -classpath ${CHUKWA_CONF_DIR}:${CLASSPATH}:${CHUKWA_CLASSPATH}:${tools} ${CLASS} $OPTS $@ &
fi
fi
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
pid=$$
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
. "$bin"/../libexec/chukwa-config.sh
pidFile=${CHUKWA_PID_DIR}/DailyChukwaRecordRolling.pid
if [ -f $pidFile ]; then
pid=`head ${pidFile}`
ChildPIDRunningStatus=`ps ax | grep DailyChukwaRecordRolling | grep -v grep | grep -o "[^ ].*" | grep ${pid} | wc -l`
if [ $ChildPIDRunningStatus -gt 0 ]; then
exit -1
fi
fi
rm -f ${pidFile}
${JAVA_HOME}/bin/java -DAPP=dailyRolling -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DCHUKWA_PID_DIR=${CHUKWA_PID_DIR} -classpath ${CHUKWA_CONF_DIR}:${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools} org.apache.hadoop.chukwa.extraction.demux.DailyChukwaRecordRolling rollInSequence true deleteRawdata true
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
pid=$$
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
. "$bin"/../libexec/chukwa-config.sh
pidFile=${CHUKWA_PID_DIR}/HourlyChukwaRecordRolling.pid
if [ -f $pidFile ]; then
pid=`head ${pidFile}`
ChildPIDRunningStatus=`ps ax | grep HourlyChukwaRecordRolling | grep -v grep | grep -o "[^ ].*" | grep ${pid} | wc -l`
if [ $ChildPIDRunningStatus -gt 0 ]; then
exit -1
fi
fi
rm -f ${pidFile}
${JAVA_HOME}/bin/java -DAPP=hourlyRolling -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_PID_DIR=${CHUKWA_PID_DIR} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -classpath ${CHUKWA_CONF_DIR}:${HADOOP_CONF_DIR}:${CLASSPATH}::${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools} org.apache.hadoop.chukwa.extraction.demux.HourlyChukwaRecordRolling rollInSequence true deleteRawdata true
================================================================================
README
================================================================================
Chukwa includes configuration files for components installed on:
> The Chukwa hadoop cluster (CC)
> The monitored source nodes (SN)
> The monitored source nodes set up as a hadoop cluster (SN-C)
To modify a configuration file, copy, rename, and edit the *.template file.
For example:
> Copy chukwa-collector-conf.xml.template to a file named
chukwa-collector-conf.xml
> Edit chukwa-collector-conf.xml to include the cluster/group name for the
source nodes
See the Chukwa Administration Guide for more information.
================================================================================
Configuration File (CC)
================================================================================
alert.template
> Modify? yes, optional
> The email address for Chukwa alerts.
--------------------------------------------
chukwa-collector-conf.xml.template
> Modify? yes
> The name node address, used by collectors to write data to HDFS
> eg: write.hdf.filesystem = hdfs://localhost:9000/
--------------------------------------------
nagios.properties.template
> Modify? yes, optional
> Nagio server that receives NSCA alert
> eg: log4j.appender.NAGIOS.Host=nagios.example.com
--------------------------------------------
aggregator.sql
> Modify? no
> SQL load scripts.
--------------------------------------------
chukwa-demux-conf.xml
> Modify? no (When available, check the Programmers Guide for more info)
> Chukwa demux settings
--------------------------------------------
chukwa-log4j.properties
> Modify? no
> Logging for Chukwa cluster (Data Processor nodes).
--------------------------------------------
commons-logging.properties
> Modify? no
> Internal log properties.
--------------------------------------------
database_create_tables.sql
> Modify? no
> SQL load scripts.
--------------------------------------------
log4j.properties
> Modify? no (When available, check the Programmers Guide for more info)
> Logging for streaming, bundle with your streaming program.
--------------------------------------------
mdl.xml
> Modify? no (When available, check the Programmers Guide for more info)
> Chukwa metrics data loader (MDL) settings.
================================================================================
Configuration File (CC, SN)
================================================================================
chukwa-env.sh.template
> Modify? yes
> Environment variables.
--------------------------------------------
default.properties
> Modify? no
> Default settings.
--------------------------------------------
build.properties
> Modify? yes, optional
> Override default settings using this file.
================================================================================
Configuration File (SN)
================================================================================
agents.template
> Modify? yes
> A list of hosts running the Chukwa agent.
--------------------------------------------
chukwa-agent-conf.xml.template
> Modify? yes
> The cluster/group name for the source nodes
> eg: chukwaAgent.tags, cluster="xyz"
--------------------------------------------
collectors.template
> Modify? yes
> A list of hosts running the Chukwa collector
> eg: http://locahost:8080/
--------------------------------------------
initial_adaptors.template
> Modify? yes, optional (When available,check the Programmers Guide for
more info)
> A list of adaptors.
--------------------------------------------
system-data-loader.properties
> Modify? no
> Metrics for streaming.
================================================================================
Configuration File (CC, SN-C)
================================================================================
hadoop-log4j.properties
> Modify? no
> Logging for Chukwa cluster (Name Node, Task Tracker, Job Tracker nodes).
> Logging for source nodes, if set up as a Hadoop cluster (same as above).
--------------------------------------------
hadoop-metrics-log4j.properties
> Modify? no
> Metrics logging for Chukwa cluster.
> Metrics logging for source nodes, if set up as a Hadoop cluster.
--------------------------------------------
hadoop-metrics.properties.template
> Modify? no
> Sample metrics for Chukwa cluster.
> Sample metrics for source nodes, if set up as a Hadoop cluster.
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>chukwaAgent.tags</name>
<value>cluster="omistack"</value>
<description>The cluster's name for this agent</description>
</property>
<property>
<name>chukwaAgent.control.port</name>
<value>9093</value>
<description>The socket port number the agent's control interface can be contacted at.</description>
</property>
<property>
<name>chukwaAgent.hostname</name>
<value>localhost</value>
<description>The hostname of the agent on this node. Usually localhost, this is used by the chukwa instrumentation agent-control interface library</description>
</property>
<property>
<name>chukwaAgent.checkpoint.name</name>
<value>chukwa_agent_checkpoint</value>
<description>the prefix to to prepend to the agent's checkpoint file(s)</description>
</property>
<property>
<name>chukwaAgent.checkpoint.dir</name>
<value>${CHUKWA_LOG_DIR}/</value>
<description>the location to put the agent's checkpoint file(s)</description>
</property>
<property>
<name>chukwaAgent.checkpoint.interval</name>
<value>5000</value>
<description>the frequency interval for the agent to do checkpoints, in milliseconds</description>
</property>
<property>
<name>chukwaAgent.sender.fastRetries</name>
<value>4</value>
<description>the number of post attempts to make to a single collector, before marking it failed</description>
</property>
<property>
<name>chukwaAgent.collector.retries</name>
<value>144000</value>
<description>the number of attempts to find a working collector</description>
</property>
<property>
<name>chukwaAgent.collector.retryInterval</name>
<value>20000</value>
<description>the number of milliseconds to wait between searches for a collector</description>
</property>
<property>
<name>syslog.adaptor.port.9095.facility.LOCAL1</name>
<value>HADOOP</value>
</property>
</configuration>
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>chukwaCollector.writerClass</name>
<value>org.apache.hadoop.chukwa.datacollection.writer.PipelineStageWriter</value>
</property>
<!-- Sequence File Writer parameters
<property>
<name>chukwaCollector.pipeline</name>
<value>org.apache.hadoop.chukwa.datacollection.writer.SocketTeeWriter,org.apache.hadoop.chukwa.datacollection.writer.SeqFileWriter</value>
</property>
-->
<!-- LocalWriter parameters
<property>
<name>chukwaCollector.localOutputDir</name>
<value>/tmp/chukwa/dataSink/</value>
<description>Chukwa local data sink directory, see LocalWriter.java</description>
</property>
<property>
<name>chukwaCollector.writerClass</name>
<value>org.apache.hadoop.chukwa.datacollection.writer.localfs.LocalWriter</value>
<description>Local chukwa writer, see LocalWriter.java</description>
</property>
-->
<!-- When writing to HBase, uncomment the following parameters. If you're running
HBase in distributed mode, you'll also need to copy your hbase-site.xml file with
your hbase.zookeeper.quorum setting to the conf/ dir. -->
<!-- HBaseWriter parameters -->
<property>
<name>chukwaCollector.pipeline</name>
<value>org.apache.hadoop.chukwa.datacollection.writer.SocketTeeWriter,org.apache.hadoop.chukwa.datacollection.writer.hbase.HBaseWriter</value>
</property>
<property>
<name>hbase.demux.package</name>
<value>org.apache.hadoop.chukwa.extraction.demux.processor</value>
<description>Demux parser class package, HBaseWriter uses this package name to validate HBase for annotated demux parser classes.</description>
</property>
<property>
<name>hbase.writer.verify.schema</name>
<value>false</value>
<description>Verify HBase Table schema with demux parser schema, log
warning if there are mismatch between hbase schema and demux parsers.
</description>
</property>
<property>