尚硅谷大数据项目【电商数仓6.0】-Flume安装-4

tar -zxf /opt/software/apache-flume-1.10.1-bin.tar.gz -C /opt/module/ mv /opt/module/apache-flume-1.10.1-bin/ /opt/module/flume vim log4j2.xml /opt/module/flume/log

xml version=“1.0” encoding=“UTF-8”?

/opt/module/flume/log

配置flume 在hadoop102节点的Flume的job目录下创建file_to_kafka.conf mkdir /opt/module/flume/job vim job/file_to_kafka.conf #定义组件 a1.sources = r1 a1.channels = c1 #配置source a1.sources.r1.type = TAILDIR a1.sources.r1.filegroups = f1 a1.sources.r1.filegroups.f1 = /opt/module/applog/log/app.* a1.sources.r1.positionFile = /opt/module/flume/taildir_position.json #配置channel a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel a1.channels.c1.kafka.bootstrap.servers = hadoop102:9092,hadoop103:9092 a1.channels.c1.kafka.topic = topic_log a1.channels.c1.parseAsFlumeEvent = false #组装 a1.sources.r1.channels = c1 vi fl.sh #!/bin/bash

检查输入参数是否为 start 或 stop

if [ -z “$1” ] || [[ “$1” != “start” && “$1” != “stop” ]]; then echo “用法: $0 {start|stop}” exit 1 fi case $1 in “start”)

启动 hadoop102 和 hadoop103 上的 Flume 采集任务

for i in hadoop102 hadoop103 do echo " ——–启动 $i 采集flume——-" ssh $i “nohup /opt/module/flume/bin/flume-ng agent -n a1 -c /opt/module/flume/conf/ -f /opt/module/flume/job/file_to_kafka.conf >/dev/null 2>&1 &” if [ $? -ne 0 ]; then echo “启动 $i 上的 Flume 失败” fi done ;; “stop”)

停止 hadoop102 和 hadoop103 上的 Flume 采集任务

for i in hadoop102 hadoop103 do echo " ——–停止 $i 采集flume——-" pid=$(ssh $i “ps -ef | grep file_to_kafka | grep -v grep | awk ‘{print $2}’”) if [ -n “$pid” ]; then ssh $i “kill $pid” if [ $? -ne 0 ]; then echo “停止 $i 上的 Flume 失败，尝试强制终止” ssh $i “kill -9 $pid” fi else echo “$i 上没有运行的 Flume 采集任务” fi done ;; esac chmod 777 fl.sh