#
# Cookbook Name:: hadoop
# Recipe:: hdp
#
# Copyright 2013, whitestar
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

::Chef::Recipe.send(:include, Helper)

install_root = '/usr/lib/hadoop'
conf_dir = '/etc/hadoop/conf'

daemons = nil
conf_files = nil

daemons_v1 = [
  'namenode',
  'datanode',
  'secondarynamenode',
  'jobtracker',
  'tasktracker'
]

daemons_v2 = [
  'hdfs-namenode',
  'hdfs-secondarynamenode',
  'hdfs-datanode',
  'yarn-resourcemanager',
  'yarn-nodemanager',
  'mapreduce-historyserver'
]

conf_files_v1 = [
  'capacity-scheduler.xml',
  'configuration.xsl',
  'core-site.xml',
  'fair-scheduler.xml',
  'hadoop-env.sh',
  'hadoop-metrics2.properties',
  'hadoop-policy.xml',
  'hdfs-site.xml',
  'hosts.include',
  'hosts.exclude',
  'log4j.properties',
  'mapred-queue-acls.xml',
  'mapred-site.xml',
  'masters',
  'slaves',
  # not supported.
  #'ssl-client.xml',
  #'ssl-server.xml'
]

conf_files_v2 = \
  conf_files_v1 \
  - [
    'configuration.xsl',
    'fair-scheduler.xml',
    'hadoop-policy.xml',  # missing!?
    'mapred-queue-acls.xml',
    'masters'
  ] \
  + [
    # not supported yet.
    #'httpfs-env.sh',
    #'httpfs-log4j.properties',
    #'httpfs-signature.secret',
    #'httpfs-site.xml',
    'mapred-env.sh',
    'yarn-env.sh',
    'yarn-site.xml'
  ] 

version, major_version, middle_version \
  = parse_hdp_version_number(node['hdp']['version'])
equivalent_apache_hadoop_middle_version \
  = get_equivalent_apache_hadoop_version_of('hdp', middle_version)

unless (('1.2.0' <= version && version < '1.4') \
  || middle_version == '2.0') then
  Chef::Application.fatal!("Non supported version: #{version}")
end

users = get_users(major_version)
# do not add any hadoop accounts by Chef.

case major_version
when '1'
  daemons = daemons_v1
  conf_files = conf_files_v1
when '2'
  daemons = daemons_v2
  conf_files = conf_files_v2
end

daemons.each {|daemon|
  package "hadoop-#{daemon}" do
    action :install
  end

  service "hadoop-#{daemon}" do
    action [:disable, :stop]
  end
}

if major_version == '1' then
  %w{
    native
    pipes
  }.each {|native_lib|
    package "hadoop-#{native_lib}" do
      action :install
    end
  }
else
  # do nothing in the ver. 2
  # native -> hadoop package
  # pipes -> hadoop package
end

link node['hadoop']['HADOOP_PREFIX'] do
  to install_root
  action [:delete, :create]
end

active_vol_nums = setup_directories(major_version)
log "This node active volumes: #{active_vol_nums}"

alt_conf_path = node['hadoop']['hdp']['alternatives']['hadoop-conf']['path']
alt_conf_priority \
  = node['hadoop']['hdp']['alternatives']['hadoop-conf']['priority']
bash 'alternatives_hadoop-conf' do
  code <<-EOC
    cp -R /etc/hadoop/conf.empty #{alt_conf_path}
    alternatives --install #{conf_dir} hadoop-conf #{alt_conf_path} #{alt_conf_priority}
  EOC
  creates alt_conf_path
end

tpl_vars = {
  :active_vol_nums => active_vol_nums
}
conf_template(conf_dir, equivalent_apache_hadoop_middle_version, conf_files, tpl_vars)

# with security
if node['hadoop']['with_security'] then
  directory node['hadoop']['this.keytab.dir'] do
    owner 'root'
    group 'root'
    mode '0755'
    action :create
    recursive true
  end
  
  case major_version
  when '1'
    # task-controller, jsvc
    package 'hadoop-sbin' do
      action :install
    end
=begin
    file "#{install_root}/bin/task-controller" do
      owner 'root'
      group users[:mapred][:name]
      mode '6050'  # '4750' by the distribution?!
    end
=end
    template "#{conf_dir}/taskcontroller.cfg" do
      source "conf-#{equivalent_apache_hadoop_middle_version}/taskcontroller.cfg"
      owner 'root'
      group 'root'
      mode '0400'  # '0644' by the distribution?!
      variables({
        :active_vol_nums => active_vol_nums
      })
    end
  when '2'
    # jsvc installation is not necessary.
    # hadoop-hdfs requires bigtop-jsvc package.
    template "/etc/default/hadoop-hdfs-datanode" do
      source "etc-#{equivalent_apache_hadoop_middle_version}/default/hadoop-hdfs-datanode"
      owner 'root'
      group 'root'
      mode '0644'
    end

    file "/usr/lib/hadoop-yarn/bin/container-executor" do
      owner 'root'
      group users[:yarn][:name]
      mode '6050'
    end
    
    template "#{conf_dir}/container-executor.cfg" do
      source "etc-#{equivalent_apache_hadoop_middle_version}/hadoop/container-executor.cfg"
      owner 'root'
      group 'root'
      mode '0400'
      variables({
        :active_vol_nums => active_vol_nums
      })
    end
  end
end

case major_version
when '1'
  log <<-EOM
Note:
You must initialize HDFS in the first installation:
  $ sudo -u hdfs hadoop namenode -format
  $ sudo service hadoop-namenode start
  $ sudo service hadoop-datanamenode start
  $ sudo -u hdfs hadoop fs -chown hdfs:hdfs /
  $ sudo -u hdfs hadoop fs -chmod 755 /
  $ sudo -u hdfs hadoop fs -mkdir /user
  $ sudo -u hdfs hadoop fs -mkdir #{node['grid']['vol_root']}/0/var/lib/mapred
  $ sudo -u hdfs hadoop fs -chown mapred:mapred #{node['grid']['vol_root']}/0/var/lib/mapred
  EOM

  examples_jar = '/usr/lib/hadoop/hadoop-examples.jar'
when '2'
  log <<-EOM
Note:
You must initialize HDFS in the first installation:
  $ sudo -u hdfs hdfs namenode -format
  $ sudo service hadoop-hdfs-namenode start
  $ sudo service hadoop-hdfs-datanamenode start
  $ sudo -u hdfs hadoop fs -chown hdfs:hdfs /
  $ sudo -u hdfs hadoop fs -chmod 755 /
  $ sudo -u hdfs hadoop fs -mkdir /user
  $ sudo -u hdfs hadoop fs -mkdir -p #{node['grid']['vol_root']}/0/var/log/yarn/nm
  $ sudo -u hdfs hadoop fs -chown yarn:hadoop #{node['grid']['vol_root']}/0/var/log/yarn/nm
  $ sudo -u hdfs hadoop fs -chmod 1777        #{node['grid']['vol_root']}/0/var/log/yarn/nm
  $ sudo -u hdfs hadoop fs -mkdir -p #{node['grid']['vol_root']}/0/var/lib/mapred/history
  $ sudo -u hdfs hadoop fs -chown -R mapred:hadoop #{node['grid']['vol_root']}/0/var/lib/mapred
  $ sudo -u hdfs hadoop fs -chmod -R 755           #{node['grid']['vol_root']}/0/var/lib/mapred
  EOM

  examples_jar = "/usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples-<version>.jar"
end

if node['hadoop']['with_security'] then
  log <<-EOM
Note:
Example MapReduce job execution:
  $ sudo -u alice kinit
  Password for alice@LOCALDOMAIN: 
  $ sudo -u alice hadoop jar #{examples_jar} pi \\
  > -D mapreduce.job.acl-view-job=* -D mapreduce.job.acl-modify-job=alice 5 10
  EOM
else
  log <<-EOM
Note:
Example MapReduce job execution:
  $ sudo adduser alice
  $ sudo -u hdfs hadoop fs -mkdir /user/alice
  $ sudo -u hdfs hadoop fs -chown alice:alice /user/alice
  $ sudo -u alice hadoop jar #{examples_jar} pi 5 10
  EOM
end
