大数据的持续搭建二(容器化)
)
一clickhouse的搭建
1.将需要的包导入master节点并且解压
[root@localhost ~]# docker cp clickhouse-client-21.9.4.35.tgz master:/root[root@localhost ~]# docker cp clickhouse-common-static-21.9.4.35.tgz master:/root[root@localhost ~]# docker cp clickhouse-common-static-dbg-21.9.4.35.tgz master:/root[root@localhost ~]# docker cp clickhouse-server-21.9.4.35.tgz master:/root[root@master ~]# mkdir /opt/module/clickhouse[root@master ~]# tar -zxvf clickhouse-client-21.9.4.35.tgz -C /opt/module/clickhouse/[root@master ~]# tar -zxvf clickhouse-common-static-21.9.4.35.tgz -C /opt/module/clickhouse/[root@master ~]# tar -zxvf clickhouse-common-static-dbg-21.9.4.35.tgz -C /opt/module/clickhouse/[root@master ~]# tar -zxvf clickhouse-server-21.9.4.35.tgz -C /opt/module/clickhouse/2.安装各个组件
[root@master ~]# cd /opt/module/clickhouse/[root@master clickhouse]# sh clickhouse-client-21.9.4.35/install/doinst.sh[root@master clickhouse]# sh clickhouse-common-static-21.9.4.35/install/doinst.sh[root@master clickhouse]# sh clickhouse-common-static-dbg-21.9.4.35/install/doinst.sh[root@master clickhouse]# sh clickhouse-server-21.9.4.35/install/doinst.shWillsetulimitsforclickhouse userin/etc/security/limits.d/clickhouse.conf. Creating config directory /etc/clickhouse-server/config.d that is usedfortweaks of main server configuration. Creating config directory /etc/clickhouse-server/users.d that is usedfortweaks ofusersconfiguration. Configfile/etc/clickhouse-server/config.xml already exists, will keep it and extract path info from it. /etc/clickhouse-server/config.xml has /var/lib/clickhouse/ as data path. /etc/clickhouse-server/config.xml has /var/log/clickhouse-server/ as log path. Users configfile/etc/clickhouse-server/users.xml already exists, will keep it and extractusersinfo from it.chown--recursiveclickhouse:clickhouse'/etc/clickhouse-server'Creating log directory /var/log/clickhouse-server/. Creating data directory /var/lib/clickhouse/. Creating pid directory /var/run/clickhouse-server.chown--recursiveclickhouse:clickhouse'/var/log/clickhouse-server/'chown--recursiveclickhouse:clickhouse'/var/run/clickhouse-server'chownclickhouse:clickhouse'/var/lib/clickhouse/'groupadd-rclickhouse-bridgeuseradd-r--shell/bin/false --home-dir /nonexistent-gclickhouse-bridge clickhouse-bridgechown--recursiveclickhouse-bridge:clickhouse-bridge'/usr/bin/clickhouse-odbc-bridge'chown--recursiveclickhouse-bridge:clickhouse-bridge'/usr/bin/clickhouse-library-bridge'Enter passwordfordefault user: Passwordfordefault user is savedinfile/etc/clickhouse-server/users.d/default-password.xml. Setting capabilitiesforclickhouse binary. This is optional. Allow server to accept connections from the network(default is localhost only),[y/N]: y The choice is savedinfile/etc/clickhouse-server/config.d/listen.xml. ClickHouse has been successfully installed. Start clickhouse-server with:sudoclickhouse start Start clickhouse-client with: clickhouse-client--passwordCreated symlink from /etc/systemd/system/multi-user.target.wants/clickhouse-server.service to /etc/systemd/system/clickhouse-server.service.3.配置相关环境
[root@master clickhouse-server]# cd /opt/module/clickhouse/clickhouse-server-21.9.4.35/etc/clickhouse-server[root@master clickhouse-server]# vim config.xml/listen_host<listen_host>::</listen_host><listen_host>0.0.0.0</listen_host /9000#9000换成9001/9000#再一次#9000换成9001二.flume的搭建
1.将需要的包导入master节点并且解压
[root@localhost ~]# docker cp apache-flume-1.9.0-bin.tar.gz master:/root[root@master module]# tar -zxvf apache-flume-1.9.0-bin.tar.gz -C /opt/module/[root@master module]# mv apache-flume-1.9.0-bin/ flume2.配置环境变量
[root@master module]# vim /etc/profileexportFLUME_HOME=/opt/module/flumeexportPATH=$PATH:$FLUME_HOME/bin[root@master module]# source /etc/profile[root@master module]# flume-ng versionFlume1.9.0 Source code repository: https://git-wip-us.apache.org/repos/asf/flume.git Revision: d4fcab4f501d41597bc616921329a4339f73585e Compiled by fszabo on Mon Dec1720:45:25 CET2018Fromsourcewith checksum 35db629a3bda49d23e9b3690c80737f93.配置相关环境
[root@master module]#cd /opt/module/flume/conf[root@master conf]# vim 1.confa1.sources=r1 a1.channels=c1 a1.sinks=k1 a1.sources.r1.type=TAILDIR a1.sources.r1.positionFile=/opt/module/hadoop.json a1.sources.r1.filegroups=f1 a1.sources.r1.filegroups.f1=/opt/module/hadoop-3.2.1/logs/.*(namenode|datanode).* a1.sources.r1.fileHeader=truea1.channels.c1.type=memory a1.channels.c1.capacity=1000a1.channels.c1.transactionCapacity=100a1.sinks.k1.type=hdfs a1.sinks.k1.hdfs.path=tmp/flume/ a1.sinks.k1.hdfs.writeFormat=Text a1.sinks.k1.hdfs.fileType=DataStream a1.sinks.k1.channel=c1 a1.sources.r1.channels=c1[root@master conf]# cp /opt/module/hadoop-3.2.1/etc/hadoop/hdfs-site.xml /opt/module/flume/conf/[root@master conf]# cp /opt/module/hadoop-3.2.1/etc/hadoop/core-site.xml /opt/module/flume/conf/[root@master conf]# cp flume-env.sh.template flume-env[root@master conf]# vim flume-envexportJAVA_HOME=/opt/module/jdk1.8.0_371[root@master conf]# rm -rf /opt/module/flume/lib/guava-11.0.2.jar[root@master conf]# cp /opt/module/hadoop-3.2.1/share/hadoop/common/lib/guava-27.0-jre.jar /opt/module/flume/lib/4.测试
[root@master flume]# ./bin/flume-ng agent -n a1 -c conf/ -f conf/1.conf -Dflume.root.logger=INFO,consoleSINK, name: k1. sink.batch.underflow==02025-10-2115:17:58,079(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:177)]Shutdown Metricfortype: SINK, name: k1. sink.channel.read.fail==02025-10-2115:17:58,079(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:177)]Shutdown Metricfortype: SINK, name: k1. sink.connection.closed.count==11462025-10-2115:17:58,079(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:177)]Shutdown Metricfortype: SINK, name: k1. sink.connection.creation.count==11462025-10-2115:17:58,079(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:177)]Shutdown Metricfortype: SINK, name: k1. sink.connection.failed.count==02025-10-2115:17:58,079(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:177)]Shutdown Metricfortype: SINK, name: k1. sink.event.drain.attempt==65792025-10-2115:17:58,079(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:177)]Shutdown Metricfortype: SINK, name: k1. sink.event.drain.sucess==65002025-10-2115:17:58,079(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:177)]Shutdown Metricfortype: SINK, name: k1. sink.event.write.fail==12025-10-2115:17:58,079(agent-shutdown-hook)[INFO - org.apache.flume.node.Application.stopAllComponents(Application.java:149)]Stopping Channel c12025-10-2115:17:58,080(agent-shutdown-hook)[INFO - org.apache.flume.lifecycle.LifecycleSupervisor.unsupervise(LifecycleSupervisor.java:169)]Stopping component: org.apache.flume.channel.MemoryChannel{name: c1}2025-10-2115:17:58,080(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:149)]Component type: CHANNEL,name: c1 stopped2025-10-2115:17:58,080(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:155)]Shutdown Metricfortype: CHANNEL, name: c1. channel.start.time==17610597823492025-10-2115:17:58,080(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:161)]Shutdown Metricfortype: CHANNEL, name: c1. channel.stop.time==17610598780802025-10-2115:17:58,080(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:177)]Shutdown Metricfortype: CHANNEL, name: c1. channel.capacity==10002025-10-2115:17:58,080(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:177)]Shutdown Metricfortype: CHANNEL, name: c1. channel.current.size==7552025-10-2115:17:58,080(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:177)]Shutdown Metricfortype: CHANNEL, name: c1. channel.event.put.attempt==73552025-10-2115:17:58,080(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:177)]Shutdown Metricfortype: CHANNEL, name: c1. channel.event.put.success==72552025-10-2115:17:58,080(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:177)]Shutdown Metricfortype: CHANNEL, name: c1. channel.event.take.attempt==65802025-10-2115:17:58,080(agent-shutdown-hook)[INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.stop(MonitoredCounterGroup.java:177)]Shutdown Metricfortype: CHANNEL, name: c1. channel.event.take.success==65002025-10-2115:17:58,080(agent-shutdown-hook)[INFO - org.apache.flume.lifecycle.LifecycleSupervisor.stop(LifecycleSupervisor.java:78)]Stopping lifecycle supervisor102025-10-2115:17:58,080(lifecycleSupervisor-1-8)[INFO - org.apache.flume.lifecycle.LifecycleSupervisor$MonitorRunnable.run(LifecycleSupervisor.java:228)]Component has already been stopped SinkRunner:{policy:org.apache.flume.sink.DefaultSinkProcessor@22ae413b counterGroup:{name:null counters:{runner.deliveryErrors=1,runner.backoffs.consecutive=0}}}2025-10-2115:17:58,083(agent-shutdown-hook)[INFO - org.apache.flume.node.PollingPropertiesFileConfigurationProvider.stop(PollingPropertiesFileConfigurationProvider.java:84)]Configuration provider stopping二.kafka的搭建
1.将需要的包导入master节点并且解压
[root@localhost ~]# docker cp kafka_2.12-2.4.1.tgz master:/root[root@master ~]# tar -zxvf kafka_2.12-2.4.1.tgz -C /opt/module/[root@master module]# mv kafka_2.12-2.4.1/ kafka2.配置环境变量
[root@master module]# vim /etc/profileexportKAFKA_HOME=/opt/module/kafkaexportPATH=$PATH:$KAFKA_HOME/bin[root@master module]# source /etc/profile[root@master module]# kafka-server-start.sh --version[2025-10-2115:33:36,980]INFO Registered kafka:type=kafka.Log4jController MBean(kafka.utils.Log4jControllerRegistration$)2.4.1(Commit:c57222ae8cd7866b)3.配置相关环境
[root@master module]#cd /opt/module/kafka/config[root@master config]# vim server.propertieslisteners=PLAINTEXT://172.17.0.3:9092advertised.listeners=PLAINTEXT://master:9092zookeeper.connect=master:2181broker.id=04.把master节点的传给其他两个节点
改那两个节点的配置文件更master一样ip,主机名broker.id=0
5.测试
[root@master kafka]# ./bin/kafka-server-start.sh config/server.properties[2025-10-2116:33:43,725]INFO[ReplicaFetcherManager on broker0]shutting down(kafka.server.ReplicaFetcherManager)[2025-10-2116:33:43,727]INFO[ReplicaFetcherManager on broker0]shutdowncompleted(kafka.server.ReplicaFetcherManager)[2025-10-2116:33:43,728]INFO[ReplicaAlterLogDirsManager on broker0]shutting down(kafka.server.ReplicaAlterLogDirsManager)[2025-10-2116:33:43,728]INFO[ReplicaAlterLogDirsManager on broker0]shutdowncompleted(kafka.server.ReplicaAlterLogDirsManager)[2025-10-2116:33:43,728]INFO[ExpirationReaper-0-Fetch]: Shutting down(kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)[2025-10-2116:33:43,914]INFO[ExpirationReaper-0-Fetch]: Stopped(kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)[2025-10-2116:33:43,915]INFO[ExpirationReaper-0-Fetch]: Shutdown completed(kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)[2025-10-2116:33:43,916]INFO[ExpirationReaper-0-Produce]: Shutting down(kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)[2025-10-2116:33:43,923]INFO[ExpirationReaper-0-Produce]: Stopped(kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)[2025-10-2116:33:43,923]INFO[ExpirationReaper-0-Produce]: Shutdown completed(kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)[2025-10-2116:33:43,924]INFO[ExpirationReaper-0-DeleteRecords]: Shutting down(kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)[2025-10-2116:33:44,117]INFO[ExpirationReaper-0-DeleteRecords]: Stopped(kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)[2025-10-2116:33:44,117]INFO[ExpirationReaper-0-DeleteRecords]: Shutdown completed(kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)[2025-10-2116:33:44,118]INFO[ExpirationReaper-0-ElectLeader]: Shutting down(kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)[2025-10-2116:33:44,123]INFO[ExpirationReaper-0-ElectLeader]: Stopped(kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)[2025-10-2116:33:44,124]INFO[ExpirationReaper-0-ElectLeader]: Shutdown completed(kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)[2025-10-2116:33:44,133]INFO[ReplicaManagerbroker=0]Shut down completely(kafka.server.ReplicaManager)[2025-10-2116:33:44,134]INFO Shutting down.(kafka.log.LogManager)[2025-10-2116:33:44,189]INFO Shutdown complete.(kafka.log.LogManager)[2025-10-2116:33:44,210]INFO[ZooKeeperClient Kafka server]Closing.(kafka.zookeeper.ZooKeeperClient)[2025-10-2116:33:44,323]INFO Session: 0x10022ef55860000 closed(org.apache.zookeeper.ZooKeeper)[2025-10-2116:33:44,325]INFO[ZooKeeperClient Kafka server]Closed.(kafka.zookeeper.ZooKeeperClient)[2025-10-2116:33:44,325]INFO EventThread shut downforsession: 0x10022ef55860000(org.apache.zookeeper.ClientCnxn)[2025-10-2116:33:44,326]INFO[ThrottledChannelReaper-Fetch]: Shutting down(kafka.server.ClientQuotaManager$ThrottledChannelReaper)[2025-10-2116:33:44,551]INFO[ThrottledChannelReaper-Fetch]: Stopped(kafka.server.ClientQuotaManager$ThrottledChannelReaper)[2025-10-2116:33:44,553]INFO[ThrottledChannelReaper-Fetch]: Shutdown completed(kafka.server.ClientQuotaManager$ThrottledChannelReaper)[2025-10-2116:33:44,554]INFO[ThrottledChannelReaper-Produce]: Shutting down(kafka.server.ClientQuotaManager$ThrottledChannelReaper)[2025-10-2116:33:45,550]INFO[ThrottledChannelReaper-Produce]: Stopped(kafka.server.ClientQuotaManager$ThrottledChannelReaper)[2025-10-2116:33:45,551]INFO[ThrottledChannelReaper-Produce]: Shutdown completed(kafka.server.ClientQuotaManager$ThrottledChannelReaper)[2025-10-2116:33:45,552]INFO[ThrottledChannelReaper-Request]: Shutting down(kafka.server.ClientQuotaManager$ThrottledChannelReaper)[2025-10-2116:33:46,546]INFO[ThrottledChannelReaper-Request]: Stopped(kafka.server.ClientQuotaManager$ThrottledChannelReaper)[2025-10-2116:33:46,546]INFO[ThrottledChannelReaper-Request]: Shutdown completed(kafka.server.ClientQuotaManager$ThrottledChannelReaper)[2025-10-2116:33:46,547]INFO[SocketServerbrokerId=0]Shutting down socket server(kafka.network.SocketServer)[2025-10-2116:33:46,577]INFO[SocketServerbrokerId=0]Shutdown completed(kafka.network.SocketServer)[2025-10-2116:33:46,582]INFO[KafkaServerid=0]shut down completed(kafka.server.KafkaServer)同时进行
[root@master kafka]# ./bin/kafka-topics.sh --create --zookeeper master:2181 --replication-factor 2 --partitions 2 --topic installtopicErrorwhileexecuting topiccommand:Replication factor:2larger than available brokers:1.[2025-10-2116:56:14,292]ERROR org.apache.kafka.common.errors.InvalidReplicationFactorException: Replication factor:2larger than available brokers:1.(kafka.admin.TopicCommand$)[root@master kafka]# jps57328QuorumPeerMain2753DataNode61330Jps2615NameNode60187Kafka2956SecondaryNameNode24860Main3245ResourceManager3391NodeManager四.zookeeper的搭建
1.将需要的包导入master节点并且解压
[root@localhost ~]# docker cp apache-zookeeper-3.5.7-bin.tar.gz master:/root[root@master ~]# tar -zxvf apache-zookeeper-3.5.7-bin.tar.gz -C /opt/module/[root@master module]# mv apache-zookeeper-3.5.7-bin/ zookeeper2.配置环境变量
[root@master module]# vim /etc/profileexportZOOKEEPER_HOME=/opt/module/zookeeperexportPATH=$PATH:$ZOOKEEPER_HOME/bin[root@master module]# source /etc/profile3.配置相关环境
[root@master module]# cd /opt/module/zookeeper/conf/[root@master conf]# cp zoo_sample.cfg zoo.cfg[root@master conf]# vim zoo.cfgdataDir=/opt/module/zookeeper/zkDataserver.1=master:2888:3888server.2=slave1:2888:3888server.3=slave2:2888:3888[root@master conf]# cd ..[root@master zookeeper]# mkdir zkData[root@master zookeeper]# cd zkData/[root@master zkData]# vim myid14.把master节点的传给其他两个节点
5.启动
[root@master zkData]# zkServer.sh startZooKeeper JMX enabled by default Using config: /opt/module/zookeeper/bin/../conf/zoo.cfg Starting zookeeper... STARTED#在另外两个从节点改zkData文件分别为2,3然后启动一样的都
)
