#
# Cookbook Name:: hadoop
# Recipe:: hdp
#
# Copyright 2013, whitestar
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

::Chef::Recipe.send(:include, Commons::PackageUtils)

::Chef::Recipe.send(:include, Hadoop::Helper)

install_root = '/usr/lib/hadoop'
conf_dir = '/etc/hadoop/conf'

daemons = nil
conf_files = nil

daemons_v1 = [
  'namenode',
  'datanode',
  'secondarynamenode',
  'jobtracker',
  'tasktracker'
]

daemons_v2 = [
  'hdfs-namenode',
  'hdfs-secondarynamenode',
  'hdfs-datanode',
  'yarn-resourcemanager',
  'yarn-nodemanager',
  'mapreduce-historyserver'
]

conf_files_v1 = [
  'capacity-scheduler.xml',
  'configuration.xsl',
  'core-site.xml',
  'fair-scheduler.xml',
  'hadoop-env.sh',
  'hadoop-metrics2.properties',
  'hadoop-policy.xml',
  'hdfs-site.xml',
  'hosts.include',
  'hosts.exclude',
  'log4j.properties',
  'mapred-queue-acls.xml',
  'mapred-site.xml',
  'masters',
  'slaves',
  # not supported.
  #'ssl-client.xml',
  #'ssl-server.xml'
]

conf_files_v2 = \
  conf_files_v1 \
  - [
    'configuration.xsl',
    'fair-scheduler.xml',
    'hadoop-policy.xml',  # missing!?
    'mapred-queue-acls.xml',
    'masters'
  ] \
  + [
    # TODO: -> hdp_httpfs recipe. (hadoop-httpfs rpm package)
    #'httpfs-env.sh',
    #'httpfs-log4j.properties',
    #'httpfs-signature.secret',
    #'httpfs-site.xml',
    'mapred-env.sh',
    'yarn-env.sh',
    'yarn-site.xml'
  ] 

version, major_version, middle_version \
  = parse_hdp_version_number(node['hdp']['version'])
equivalent_apache_hadoop_middle_version \
  = get_equivalent_apache_hadoop_version_of('hdp', node['hdp']['version'])

users = get_users(major_version)
# do not add any hadoop accounts by Chef.

case major_version
when '1'
  daemons = daemons_v1
  if middle_version >= '1.3' then
    daemons.push('historyserver')
  end
  conf_files = conf_files_v1
when '2'
  daemons = daemons_v2
  conf_files = conf_files_v2
end

daemons.each {|daemon|
  package "hadoop-#{daemon}" do
    action node['hadoop']['packages']['action']
  end

  service "hadoop-#{daemon}" do
    action node['hadoop']['services']['action']
  end
}

if major_version == '1' then
  %w{
    libhdfs
    native
    pipes
  }.each {|native_lib|
    package "hadoop-#{native_lib}" do
      action node['hadoop']['packages']['action']
    end
  }
else
  # do nothing in the ver. 2
  # native -> hadoop package
  # pipes -> hadoop package
end

if node['hadoop']['with_hadoop_lzo'] then
  %w{
    lzo
    lzo-native
  }.each {|lzo_pkg|
    package "hadoop-#{lzo_pkg}" do
      action node['hadoop']['packages']['action']
    end
  }
end

if node['hadoop']['io.compression.codec.bzip2.library'] == 'system-native' \
  && equivalent_apache_hadoop_middle_version >= '2.1' then
  package get_libbz2_pkg_name do
    action :install
  end
end

link node['hadoop']['HADOOP_PREFIX'] do
  to install_root
  action [:delete, :create]
end

active_vol_nums = setup_directories(major_version)
log "This node active volumes: #{active_vol_nums}"

alt_conf_path = node['hadoop']['hdp']['alternatives']['hadoop-conf']['path']
alt_conf_priority \
  = node['hadoop']['hdp']['alternatives']['hadoop-conf']['priority']
bash 'alternatives_hadoop-conf' do
  code <<-EOC
    cp -R /etc/hadoop/conf.empty #{alt_conf_path}
    alternatives --install #{conf_dir} hadoop-conf #{alt_conf_path} #{alt_conf_priority}
  EOC
  creates alt_conf_path
end

tpl_vars = {
  :active_vol_nums => active_vol_nums
}
conf_template(conf_dir, equivalent_apache_hadoop_middle_version, conf_files, tpl_vars)

# with security
if node['hadoop']['with_security'] then
  directory node['hadoop']['this.keytab.dir'] do
    owner 'root'
    group 'root'
    mode '0755'
    action :create
    recursive true
  end
  
  case major_version
  when '1'
    # task-controller, jsvc
    package 'hadoop-sbin' do
      action node['hadoop']['packages']['action']
    end
=begin
    file "#{install_root}/bin/task-controller" do
      owner 'root'
      group users[:mapred][:name]
      mode '6050'  # '4750' by the distribution?!
    end
=end
    template "#{conf_dir}/taskcontroller.cfg" do
      source "conf-#{equivalent_apache_hadoop_middle_version}/taskcontroller.cfg"
      owner 'root'
      group 'root'
      mode '0400'  # '0644' by the distribution?!
      variables({
        :active_vol_nums => active_vol_nums
      })
    end
  when '2'
    # jsvc installation is not necessary.
    # hadoop-hdfs requires bigtop-jsvc package.
    template "/etc/default/hadoop-hdfs-datanode" do
      source "etc-#{equivalent_apache_hadoop_middle_version}/default/hadoop-hdfs-datanode"
      owner 'root'
      group 'root'
      mode '0644'
    end

    file "/usr/lib/hadoop-yarn/bin/container-executor" do
      owner 'root'
      group users[:yarn][:name]
      mode '6050'
    end
    
    template "#{conf_dir}/container-executor.cfg" do
      source "etc-#{equivalent_apache_hadoop_middle_version}/hadoop/container-executor.cfg"
      owner 'root'
      group 'root'
      mode '0400'
      variables({
        :active_vol_nums => active_vol_nums
      })
    end

    if version >= '2.0.0.2' then
      setup_cgroup(equivalent_apache_hadoop_middle_version)
    end
  end
end

# Note: update the following template (hadoop_pseudo_distributed_init.sh)
#       if this section will be modified.
case major_version
when '1'
  log <<-EOM
Note:
You must initialize HDFS in the first installation:
  $ sudo -u hdfs hadoop namenode -format
  $ sudo service hadoop-namenode start
  $ sudo service hadoop-datanode start
  $ sudo -u hdfs hadoop fs -chown hdfs:hdfs /
  $ sudo -u hdfs hadoop fs -chmod 755 /
  $ sudo -u hdfs hadoop fs -mkdir /user
  $ sudo -u hdfs hadoop fs -mkdir #{node['grid']['vol_root']}/0/var/lib/mapred
  $ sudo -u hdfs hadoop fs -chown mapred:mapred #{node['grid']['vol_root']}/0/var/lib/mapred
  EOM

  examples_jar = '/usr/lib/hadoop/hadoop-examples.jar'
when '2'
  log <<-EOM
Note:
You must initialize HDFS in the first installation:
  $ sudo -u hdfs hdfs namenode -format
  $ sudo service hadoop-hdfs-namenode start
  $ sudo service hadoop-hdfs-datanode start
  $ sudo -u hdfs hadoop fs -chown hdfs:hdfs /
  $ sudo -u hdfs hadoop fs -chmod 755 /
  $ sudo -u hdfs hadoop fs -mkdir /user
  $ sudo -u hdfs hadoop fs -mkdir -p #{node['grid']['vol_root']}/0/var/log/yarn/nm
  $ sudo -u hdfs hadoop fs -chown yarn:hadoop #{node['grid']['vol_root']}/0/var/log/yarn/nm
  $ sudo -u hdfs hadoop fs -chmod 1777        #{node['grid']['vol_root']}/0/var/log/yarn/nm
  $ sudo -u hdfs hadoop fs -mkdir -p #{node['grid']['vol_root']}/0/var/lib/mapred/history
  $ sudo -u hdfs hadoop fs -chown -R mapred:hadoop #{node['grid']['vol_root']}/0/var/lib/mapred
  $ sudo -u hdfs hadoop fs -chmod -R 755           #{node['grid']['vol_root']}/0/var/lib/mapred
  EOM

  examples_jar = "/usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples-<version>.jar"
end

template "#{node['grid']['app_root']}/sbin/hadoop_pseudo_distributed_init.sh" do
  source "grid/usr/sbin/hadoop_pseudo_distributed_init.hdp.sh"
  owner 'root'
  group 'root'
  mode '0755'
  variables({
    :major_version => major_version,
  })
end

if node['hadoop']['with_security'] then
  log <<-EOM
Note:
Example MapReduce job execution:
  $ sudo -u alice kinit
  Password for alice@LOCALDOMAIN: 
  $ sudo -u alice hadoop jar #{examples_jar} pi \\
  > -D mapreduce.job.acl-view-job=* -D mapreduce.job.acl-modify-job=alice 5 10
  EOM
else
  log <<-EOM
Note:
Example MapReduce job execution:
  $ sudo adduser alice
  $ sudo -u hdfs hadoop fs -mkdir /user/alice
  $ sudo -u hdfs hadoop fs -chown alice:alice /user/alice
  $ sudo -u alice hadoop jar #{examples_jar} pi 5 10
  EOM
end
