离线部署CDH5.16.2

在实践了CDH6的离线部署之后,发现Cloudera的官方教程是基于CDH5.10的。同时,CDH6的内存占用比较高,我的机器有点遭不住,所以又把CDH5的部署实施走了一遍。

模板部署

  1. 安装CentOS7.10,并完成yum升级;
  2. 所有主机时间与阿里云时间服务器同步;
  3. 关闭selinux和firewalld;
  4. 安装JDK和mysql-connect;
[root@localhost ~]# wget https://repo.huaweicloud.com/java/jdk/8u202-b08/jdk-8u202-linux-x64.rpm
[root@localhost ~]# yum localinstall jdk-8u202-linux-x64.rpm
[root@localhost ~]# mv mysql-connector-java-5.1.39-bin.jar /usr/share/java/mysql-connector-java.jar 
  1. 关闭透明大页;
[root@localhost ~]# vim /etc/rc.d/rc.local

# 在最后一行添加如下内容
if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi

if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi

# 赋予rc.local具有可执行权限
[root@localhost ~]# chmod +x /etc/rc.d/rc.local
[root@localhost ~]# sh /etc/rc.d/rc.local

# 检查
[root@localhost ~]# cat /sys/kernel/mm/transparent_hugepage/defrag
always madvise [never]
[root@localhost ~]# cat /sys/kernel/mm/transparent_hugepage/enabled
always madvise [never]
  1. 调整swap策略
[root@localhost ~]# echo 10 > /proc/sys/vm/swappiness
[root@localhost ~]# echo vm.swappiness = 10 >> /etc/sysctl.conf
  1. 修改文件句柄数
[root@localhost ~]# vim /etc/security/limits.conf
# 添加如下内容
* soft nofile 100000
* hard nofile 100000
  1. 收尾
[root@localhost ~]# sys-unconfig
  1. 下载CM和CDH

CM下载地址

CDH下载地址

Agent部署

  1. 配置主机ssh信任
  2. 安装ansible

# 安装 [root@elephant ~]# yum install -y ansible [root@elephant ~]# ls CDH-5.16.2-1.cdh5.16.2.p0.8-el5.parcel.sha1 CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel Mail anaconda-ks.cfg cloudera-manager-agent-5.16.2-1.cm5162.p0.7.el7.x86_64.rpm cloudera-manager-daemons-5.16.2-1.cm5162.p0.7.el7.x86_64.rpm cloudera-manager-server-5.16.2-1.cm5162.p0.7.el7.x86_64.rpm cloudera-manager-server-db-2-5.16.2-1.cm5162.p0.7.el7.x86_64.rpm enterprise-debuginfo-5.16.2-1.cm5162.p0.7.el7.x86_64.rpm hadoop.tar.gz jdk-8u202-linux-x64.rpm manifest.json mysql-connector-java-5.1.39-bin.jar [root@elephant ~]# mkdir client [root@elephant ~]# mv cloudera-manager-daemons-5.16.2-1.cm5162.p0.7.el7.x86_64.rpm client/ [root@elephant ~]# mv cloudera-manager-agent-5.16.2-1.cm5162.p0.7.el7.x86_64.rpm client/ [root@elephant ~]# tar zcvf client.tar.gz client/
  1. 分发
[root@elephant ~]# ansible all -m copy -a 'src=/root/client.tar.gz dest=/root/'
[root@elephant ~]# ansible all -a 'tar zxf /root/client.tar.gz'
  1. 安装
[root@elephant ~]# ansible all -a 'yum localinstall /root/client/cloudera-manager-daemons-5.16.2-1.cm5162.p0.7.el7.x86_64.rpm -y'
[root@elephant ~]# ansible all -a 'yum localinstall /root/client/cloudera-manager-agent-5.16.2-1.cm5162.p0.7.el7.x86_64.rpm -y'

# 修改配置文件
[root@elephant ~]# sed -i ':s/localhost/lion/g' /etc/cloudera-scm-agent/config.ini
[root@elephant ~]# ansible lion,tiger,horse,monkey -m copy -a 'src=/etc/cloudera-scm-agent/config.ini dest=/etc/cloudera-scm-agent/'

# 重启服务
[root@elephant ~]# ansible all -a 'systemctl enable cloudera-scm-agent --now'
[root@elephant ~]# ansible all -a 'systemctl restart cloudera-scm-agent'

管理主机部署

  1. 安装
[root@elephant ~]# ansible lion -m copy -a 'src=/root/cloudera-manager-server-5.16.2-1.cm5162.p0.7.el7.x86_64.rpm dest=/root/'
[root@elephant ~]# ansible lion -m copy -a 'src=/root/CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel dest=/opt/cloudera/parcel-repo'
[root@elephant ~]# ansible lion -m copy -a 'src=/root/manifest.json dest=/opt/cloudera/parcel-repo'
[root@elephant ~]# ansible lion -a 'yum localinstall /root/cloudera-manager-server-5.16.2-1.cm5162.p0.7.el7.x86_64.rpm -y'
  1. 数据库部署
[root@elephant ~]# ansible lion -a 'yum install -y mariadb mariadb-server'
[root@elephant ~]# ssh lion
[root@lion ~]# vim /etc/my.cnf.d/server.cnf
[mysqld]
key_buffer = 16M
key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1
max_connections = 550
server_id=1

binlog_format = mixed
read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M

# InnoDB settings
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit  = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M

[root@lion ~]# systemctl enable mariadb --now

vim cdh.sql

CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON metastore.* TO 'hive'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY 'passwd';

# 导入数据库
[root@lion ~]# mysql -uroot -p < cdh.sql

# 扩展数据库架构和CDH6有区别
[root@lion ~]# cp /usr/share/java/mysql-connector-java.jar /usr/share/cmf/lib/
[root@lion ~]# sh /usr/share/cmf/schema/scm_prepare_database.sh mysql scm scm
Enter SCM password:
JAVA_HOME=/usr/java/jdk1.8.0_202-amd64
Verifying that we can write to /etc/cloudera-scm-server
Creating SCM configuration file in /etc/cloudera-scm-server
Executing:  /usr/java/jdk1.8.0_202-amd64/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/java/postgresql-connector-java.jar:/usr/share/cmf/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /etc/cloudera-scm-server/db.properties com.cloudera.cmf.db.
[                          main] DbCommandExecutor              INFO  Successfully connected to database.
All done, your SCM database is configured correctly!

# 启动服务
[root@lion parcel-repo]# cd /opt/cloudera/parcel-repo/
[root@lion parcel-repo]# sha1sum CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel | awk '{ print $1 }' > CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel.sha
[root@lion ~]# chkconfig cloudera-scm-server on
[root@lion ~]# systemctl start cloudera-scm-server

# 检查结果
[root@lion ~]# netstat -tlnp |grep 7180
tcp        0      0 0.0.0.0:7180            0.0.0.0:*               LISTEN      3907/java 

GUI安装

接下来的安装就相对简单了,基本比CDH6简单。

start
start
start
start
start
start
start
start
start
start
start
end

离线部署CDH6.3

部署练习的环境有六台主机,主机的配置是4核心4G内存(内存还是低了,建议8G起步):

主机名 IP地址
Elephant 192.168.174.184
Lion 192.168.174.185
Horse 192.168.174.186
Monkey 192.168.174.187
Tiger 192.168.174.188

环境准备

  1. 主机环境配置

按照各个主机配置修改相应值,下面以elephant为例:

[root@localhost ~]# nmcli c m ens33 ipv4.method manual ipv4.addresses 192.168.174.184/24 ipv4.gateway 192.168.174.2 ipv4.dns 192.168.174.2
[root@localhost ~]# nmcli c d ens33 && nmcli c u ens33
[root@localhost ~]# hostnamectl set-hostname elephant
  1. 文件复制
    将离线部署CDH6.3.1的文件复制到每一台主机上。其中包括:
sujx@legion7000:/mnt/d/$ ls Cloudera
CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel
cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm
cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm
cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm
cloudera-manager-server-db-2-6.3.1-1466458.el7.x86_64.rpm
enterprise-debuginfo-6.3.1-1466458.el7.x86_64.rpm
jdk-8u202-linux-x64.rpm
manifest.json
mysql-connector-java-5.1.39-bin.jar

sujx@legion7000:/mnt/d/$ tar zcvf Cloudera cloudera.tar.gz
sujx@legion7000:/mnt/d/$ scp cloudera.tar.gz root@lion:/root/
  1. 免密登录设置

在lion主机上设置免密:

ssh-keygen -t rsa
ssh-copy-id {elephant\lion\horse\monkey\tiger}

# 使用ansible分发和管理文件,省略配置文件的建立
yum install -y ansible

ansible -m copy -a 'src=/root/cloudera.tar.gz dest=/root'

  1. 主机配置
    建立主机执行文件进行相关配置。
#!/bin/bash

# 安装必要组件
yum update -y
yum install -y mutt net-tools mlocate telnet curl wget ping vim open-vm-tools

systemctl disable firewalld.service --now
sed -i ':s/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux
updatedb && sync && ldconfig

# 配置时间服务器
cat >/etc/chrony.conf<<EOF
server ntp.aliyun.com iburst

driftfile /var/lib/chrony/drift
makestep 1.0 3
rtcsync
logdir /var/log/chrony

EOF

systemctl enable chronyd --now
timedatectl 

# 优化
# 修改swap配置
echo "vm.swappiness = 10" >> /etc/sysctl.conf
sysctl -p

# 修改大页内存的使用
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' >> /etc/rc.local
echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >> /etc/rc.local

# 安装JAVA
tar zxvf cloudera.tar.gz
yum localinstall -y Cloudera/jdk-8u202-linux-x64.rpm

java -version
echo $JAVA_HOME

# 建立安装文件源
wget https://archive.cloudera.com/cm6/6.3.1/redhat7/yum/RPM-GPG-KEY-cloudera
rpm --import RPM-GPG-KEY-cloudera
wget https://archive.cloudera.com/cm6/6.3.1/redhat7/yum/cloudera-manager.repo -O /etc/pki/rpm-gpg/RPM-GPG-KEY-cloudera
yum update -y

mkdir /usr/share/java/
cp Cloudera/mysql-connector-java-5.1.39-bin.jar  /usr/share/java/mysql-connector-java.jar

yum localinstall -y Cloudera/cloudera-manager-daemons* 
yum localinstall -y Cloudera/cloudera-manager-agent*

# 修改cloudera-scm客户端的服务器目标地址
sed -i ':s/localhost/lion/g' /etc/cloudera-scm-agent/config.ini
systemctl enable cloudera-scm-agent
systemctl restart cloudera-scm-agent

# 客户端配置完成

数据库安装

数据库将部署在Lion主机之上。
1. 数据库安装

yum install -y mariadb mariadb-server

# 增加MySQL配置文件
vim /etc/my.cnf.d/server.cnf
[mysqld]
key_buffer = 16M
key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1

max_connections = 550
server_id=1

binlog_format = mixed

read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M

# InnoDB settings
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit  = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M

# 启动数据库服务
systemctl enable mariadb --now

2. 建立数据库配置文件
vim /root/cdh.sql
CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON metastore.* TO 'hive'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY 'passwd';
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY 'passwd';

mysql -uroot -p < /root/cdh.sql
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm

### 管理节点安装
```shell
yum localinstall -y Clouder/cloudera-manager-server.6*
cp Cloudera/CDH* /opt/cloudera/parcel-repo
cp Cloudera/main* /opt/cloudera/parcel-repo
cd /opt/cloudera/parcel-repo
# wget https://archive.cloudera.com/cdh6/6.3.2/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554-el6.parcel -P /opt/cloudera/parcel-repo
# wget https://archive.cloudera.com/cdh6/6.3.2/parcels/manifest.json -P /opt/cloudera/parcel-repo
sha1sum CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel | awk '{ print $1 }' > CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha
chown -R cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo/*
systemctl enable cloudera-scm-server --now
cd

群集配置

访问Lion主机的7180端口,然后使用web-gui界面进行配置。
start
start
start
start
start
start
start
start
start
start
start
start
start
start
start
start
start
start
start
start
start