CDH6.2安装 - HelloWorld开发者社区

一、准备工作：
1、普通用户root权限准备（sudo，此处用aiot用户，sudoer）
groupadd -r test
useradd -d /home/test/ -m -s /bin/bash -c "user" -g test -p 2019 -r test
#另外特别注意：aiot及密码用户在cloudera manager server中会配置进去，作为集群管理【host添加、agent安装&卸载、parcels管理os user】，
所以不能轻易在os修改aiot用户密码，如果需要修改，运维同学需要通知cdh集群管理员同步修改cms保存的密码）
passwd test
root： vim /etc/sudoers
test ALL=(ALL) NOPASSWD: ALL
2.必备工具
sudo yum install yum-utils createrepo -y
sudo yum install ansible -y
sudo yum install httpd -y
sudo yum install systemctl -y
sudo yum install ntp -y
sudo yum install hostnamectl -y
~~~~~~~~~~~~~~~~~~~

3.集群 /etc/hosts设置
sudo vim /etc/hosts
cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.0.3 cdh3
192.168.0.4 cdh4
192.168.0.5 cdh5

#修改主机名
sudo vim /etc/sysconfig/network：
NETWORKING=yes
HOSTNAME=cdh3

reboot

4.关闭防火墙
getenforce
sudo systemctl stop firewalld
sudo systemctl disable firewalld.service
sudo vim /etc/selinux/config

-------------------------------

SELINUX=disabled

-------------------------------

5 ssh 互信（所有节点）test用户
$ mkdir .ssh
$ chmod 750 .ssh
$ ssh-keygen -t rsa
##192.168.0.3操作
$cat /home/aiot/.ssh/id_rsa.pub >> /home/aiot/.ssh/authorized_keys
$ chmod 600 ~/.ssh/authorized_keys

所有节点测试互信测试

6.配置NTP服务（所有节点）两种方法二选一

sudo yum install -y ntp
方法一时间同步主节点

----------------------------------------

主节点配置(192.168.0.3 )
sudo vim /etc/ntp.conf
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
server ntp.aliyun.com
从节点配置(192.168.0.4 192.168.0.5)
sudo vim /etc/ntp.conf
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
server 192.168.0.3 cdh3

----------------------------------------------

sudo systemctl enable ntpd
sudo systemctl start ntpd

6.所有节点禁用大透明页和vm.swappiness设置
sysctl vm.swappiness=10
vim /etc/sysctl.conf

vm.swappiness=10

echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
上面两条加入/etc/rc.local开机自启
vim /etc/rc.local
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
chmod +x /etc/rc.d/rc.local

7.JDK安装（所有节点）
sudo rpm -qa | grep java # 查询已安装的java
sudo yum remove java* # 卸载
sudo mkdir -p /usr/java
sudo tar -zxf /tmp/installer/jdk-8u181-linux-x64.tar.gz -C /usr/java/
sudo ln -s /usr/java/jdk1.8.0_181 /usr/java/jdk
sudo vim /etc/profile

export JAVA_HOME=/usr/java/jdk
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib:$CLASSPATH

sudo source /etc/profile
#如果不是root用户需要添加
~/.bashrc
~/.bash_profile

检查环境变量 echo $JAVA_HOME

所有节点以下操作
sudo mkdir -p /usr/share/java
sudo cp /data/mysql-connector-java-5.1.46.jar /usr/share/java/mysql-connector-java.jar

二#mysql5.7.20安装
sudo rpm -qa | grep mariadb
sudo rpm -e mariadb-libs-5.5.60-1.el7_5.x86_64 --nodeps
sudo yum -y install libaio
#官方下载
#sudo wget https://dev.mysql.com/get/Downloads/MySQL-5.7/mysql-5.7.20-1.el7.x86_64.rpm-bundle.tar
#解压
#sudo tar -xvf mysql-5.7.20-1.el7.x86_64.rpm-bundle.tar
#本地rpm安装
sudo rpm -ivh mysql-community-common-5.7.20-1.el7.x86_64.rpm
sudo rpm -ivh mysql-community-libs-5.7.20-1.el7.x86_64.rpm
sudo rpm -ivh mysql-community-libs-compat-5.7.20-1.el7.x86_64.rpm
sudo rpm -ivh mysql-community-client-5.7.20-1.el7.x86_64.rpm
sudo rpm -ivh mysql-community-server-5.7.20-1.el7.x86_64.rpm
#如果指定了mysql用户需要授权给mysql
没有/data/mysqldata/mysql目录
需要
sudo mkdir -p /data/mysqldata/mysql
sudo vim /etc/my.cnf:
--------------------------------
[mysqld]
datadir=/data/mysqldata/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation=READ-COMMITTED
symbolic-links=0
key_buffer_size=32M
max_allowed_packet=32M
thread_stack=256K
thread_cache_size=64
query_cache_limit=8M
query_cache_size=64M
query_cache_type=1
max_connections=550
log_bin=/data/mysqldata/mysql/mysql_binary_log
server_id=1
binlog_format=mixed
read_buffer_size=2M
read_rnd_buffer_size=16M
sort_buffer_size=8M
join_buffer_size=8M
innodb_file_per_table=1
innodb_flush_log_at_trx_commit=2
innodb_log_buffer_size=64M
innodb_buffer_pool_size=4G
innodb_thread_concurrency=8
innodb_flush_method=O_DIRECT
innodb_log_file_size=512M
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid
sql_mode=STRICT_ALL_TABLES
-----------------------------
sudo systemctl enable mysqld
#sudo systemctl stop mysqld
sudo systemctl start mysqld
sudo systemctl status mysqld -l
sudo grep 'temporary password' /var/log/mysqld.log
#过滤出初始密码然后登陆
mysql -u root -p
#修改用户密码用下面这条
ALTER USER USER() IDENTIFIED BY '2019';
#必要的库和用户、权限
grant all privileges on *.* to 'root'@'%' identified by '2019' with grant option;
create database scm DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database nas DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database actmonitor DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database rptmgn DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database navmetas DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database sentry DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

grant all privileges on *.* to 'admin'@'%' identified by '2019' with grant option;

grant all privileges on scm.* to 'scm'@'%' identified by '2019' with grant option;
grant all privileges on hive.* to 'hive'@'%' identified by '2019' with grant option;
grant all privileges on oozie.* to 'oozie'@'%' identified by '2019' with grant option;
grant all privileges on hue.* to 'hue'@'%' identified by '2019' with grant option;
grant all privileges on nas.* to 'nas'@'%' identified by '2019' with grant option;
grant all privileges on actmonitor.* to 'actmonitor'@'%' identified by '2019' with grant option;
grant all privileges on rptmgn.* to 'rptmgn'@'%' identified by '2019' with grant option;
grant all privileges on navmetas.* to 'navmetas'@'%' identified by '2019' with grant option;
grant all privileges on sentry.* to 'sentry'@'%' identified by '2019' with grant option;
flush privileges;

安装CMS 主节点 192.168.0.3
准备: CMS、CDH-Parcel internal REPO搭建 [6.2.0为例]
1）sudo yum install httpd
2）sudo vim /etc/httpd/conf/httpd.conf添加如下：
AddType application/x-gzip .gz .tgz .parcel
3)sudo systemctl start httpd
sudo systemctl enable httpd
#创建本地yum源仓库实现本地yum安装
4)sudo mkdir -p /var/www/html/cloudera-repos
####5-10步骤没有加速特别慢自己下的时候发现是在dump网站建议拷贝可以删除里面其它不用的包
###换为下面的单独下载包5）sudo proxychains4 wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/ -P /var/www/html/cloudera-repos
####5 centos选择CDH-6.2.0-1.cdh6.2.0.p0.967373-el7开头的三个下载即可

sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel -P /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel.sha1 -P /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel.sha256 -P /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/manifest.json -P /var/www/html/cloudera-repos

6）sudo proxychains4 wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/gplextras6/6.2.0/parcels/ -P /var/www/html/cloudera-repos
######6 centos选择GPLEXTRAS-6.2.0-1.gplextras6.2.0.p0.967373-el7开头的三个下载即可
7）sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cm6/6.2.0/redhat7/ -P /var/www/html/cloudera-repos
8）sudo wget https://archive.cloudera.com/cm6/6.2.0/allkeys.asc -P /var/www/html/cloudera-repos/cm6/6.2.0/
9）sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/kafka/parcels/4/ -P /var/www/html/cloudera-repos
10）sudo wget --recursive --no-parent --no-host-directories http://archive.cloudera.com/sqoop-connectors/parcels/latest/ -P /var/www/html/cloudera-repos
11)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/cdh6
12)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/gplextras6
13)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/cm6
14)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/kafka
15)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/sqoop-connectors
16）curl http://localhost/cloudera-repos/
17)sudo vim /etc/yum.repos.d/cloudera-manager6.2.0.repo

[cloudera-manager]
name=Cloudera Manager 6.2.0
baseurl=http://本机IP或者本地hosts解析/cloudera-repos/cm6/6.2.0/redhat7/yum/
gpgkey=http://本机IP或者本地hosts解析/cloudera-repos/cm6/6.2.0/redhat7/yum/RPM-GPG-KEY-cloudera
gpgcheck=1
enabled=1
autorefresh=0
type=rpm-md

17.2)sudo vim /etc/yum.repos.d/cloudera-manager-gplextras6.2.0.repo

#两模块不要一样
[cloudera-manager-gplextras]
name=Cloudera Manager gplextras 6.2.0
baseurl=http://本机IP或者本地hosts解析/cloudera-repos/gplextras6/6.2.0/redhat7/yum/
gpgkey=http://本机IP或者本地hosts解析/cloudera-repos/gplextras6/6.2.0/redhat7/yum/RPM-GPG-KEY-cloudera
gpgcheck=1
enabled=1
autorefresh=0
type=rpm-md

18)sudo vim /etc/httpd/conf/httpd.conf

<Directory "/var/www/cgi-bin">
AllowOverride None
Options FollowSymLinks
Order deny,allow
allow from all

19)sudo systemctl restart httpd
20)sudo yum clean all
21)sudo yum makecache
#执行 yum makecache 报错缺少/repodata/repomd.xml: [Errno 14] HTTP Error 404 - Not Found
#解决办法
#proxychains4 wget --recursive --no-parent --no-host-directories http://archive.cloudera.com/gplextras6/6.2.0/redhat7/yum/repodata/ -P /var/www/html/cloudera-repos
22)sudo yum update
23)sudo yum -y search cloudera-manager
24)sudo systemctl enable httpd

三）6.2.0 internal repo
#如果要求全部内网下载可以把主节点的repo直接ansible推过去[主节点一定要执行从节点可选吧查资料上说为避免尽可能的减少出错三节点都提前安装好但只需要主节点启动server端]

sudo yum install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server
echo $JAVA_HOME

#只需要主节点执行------
sudo JAVA_HOME=/usr/java/jdk /opt/cloudera/cm-agent/bin/certmanager --location /opt/cloudera/CMCA setup --configure-services
sudo -E /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm '2019'

Android进阶之旅-(NDK实战篇之OpenCV）

Android进阶之音视频技术

热门文章