2013年7月8日 星期一

Hadoop on Rocks Cluster

build hadoop from source
chmod -R 755 /state/partition1/data/hadoop/

conf/master
light

conf/slave
light
compute-0-1
...
compute-0-10


==> conf/core-site.xml <==
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Putsite-specificpropertyoverridesinthisfile. -->

<configuration>
 <property>
  <name>fs.default.name</name>
  <value>hdfs://light:54310/</value>
 </property>
 <property>
  <name>df.permissions</name>
  <value>false</value>
 </property>
</configuration>

==> conf/hdfs-site.xml <==
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
<property>
<name>dfs.name.dir</name>
<value>/state/partition1/data/hadoop/name</value>
<description> </description>
</property>
<property>
<name>dfs.data.dir</name>
<value>/state/partition1/data/hadoop/data</value>
<description> </description>
</property>
</configuration>

==> conf/mapred-site.xml <==
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
<property>
<name>mapred.job.tracker</name>
<value>light:54311</value>
</property>
</configuration>