Scribe 安装使用

Scribe 是 Facebook 开源的日志收集工具，我建议大家还是老老实实使用 ELK/EFK 或者 Splunk，不要瞎折腾 Scribe

Centos 安装 Scribe 真是累

2020 年 06 月 09 日 - 增加 Docker 容器部署 Scribe
2018 年 09 月 27 日 - 初稿

阅读原文 - https://wsgzao.github.io/post/scribe/

Scribe 简介

Scribe is a server for aggregating log data that’s streamed in real time from clients. It is designed to be scalable and reliable.ß

See the Scribe Wiki for documentation:
http://wiki.github.com/facebook/scribe

install scribe

OS: CentOS 7.x

This guide shows how to install the precompiled scribed binary and its dependencies on a new server. Care must be taken to use the required versions for boost (1.44.0), thrift (0.7.0) and libevent (1.4.10).

# install necessary tools
sudo yum install git make flex bison libtool automake openssl-devel libevent libevent-devel python-devel gcc-c++ byacc java-1.7.0-openjdk ant autoconf boost-devel

# update autoconf
# http://ftp.gnu.org/gnu/autoconf/
cd ~
sudo rpm -e --nodeps `rpm -qf /usr/bin/autoconf`
wget http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz
tar xzf autoconf-2.69.tar.gz
cd autoconf-2.69/
./configure
make
sudo make install
# make sure /usr/local/bin is in system PATH: add following line to ~/.bashrc
vi ~/.bashrc
export PATH=$PATH:/usr/local/bin

# install boost
# http://jaist.dl.sourceforge.net/project/boost/boost/
cd ~
wget http://jaist.dl.sourceforge.net/project/boost/boost/1.44.0/boost_1_44_0.tar.gz
tar xzf boost_1_44_0.tar.gz
cd boost_1_44_0/
./bootstrap.sh
sudo ./bjam install

# install thrift
cd ~
git clone https://github.com/apache/thrift.git
cd thrift/
git fetch
git branch -a
git checkout 0.7.x
./bootstrap.sh
./configure
make
cp compiler/cpp/thrifty.hh compiler/cpp/thrifty.h
make
sudo make install
cd lib/py/
sudo python setup.py install

# install fb303
cd ~/thrift/contrib/fb303/
./bootstrap.sh
./configure CPPFLAGS="-DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H"
make
sudo make install
cd py/
sudo python setup.py install

# install libevent
cd ~
wget https://github.com/downloads/libevent/libevent/libevent-1.4.10-stable.tar.gz
tar xf libevent-1.4.10-stable.tar.gz 
cd libevent-1.4.10-stable
./configure
make
make install

# Copy the scribed binary and put it in the desired directory
scp scribe.tar.gz 10.65.32.134:/tmp
scp scribed 10.65.32.134:/tmp
# unzip and mkdir
mkdir /opt/running
cd /opt/running
cp /tmp/scribe.tar.gz .
tar xzvf scribe.tar.gz
rm -f scribe.tar.gz
mkdir -p /data/gop/live/primary/gop
mkdir -p /data/gop/live/secondary/gop
ln -s /data/gop/live/primary/gop/ /data/shortcut_gop
# rm -r /data/shortcut_gop
cp /tmp/scribed /usr/local/bin/scribed
cd scribe/

[root@vm4 scribe]# cat run_scribed.sh 
#!/bin/bash

while true
do
        export LD_LIBRARY_PATH=/usr/lib:/usr/local/lib
        /usr/local/bin/scribed scribe_11315.conf 1>output/scribe.log 2>output/scribe.err
        sleep 1
done

nohup sh run_scribed.sh &

[root@sg-gop-10-65-32-134 scribe]# netstat -tlnp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:10050           0.0.0.0:*               LISTEN      55310/zabbix_agentd
tcp        0      0 0.0.0.0:38422           0.0.0.0:*               LISTEN      52781/sshd
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      1305/sshd
tcp6       0      0 :::10050                :::*                    LISTEN      55310/zabbix_agentd
tcp6       0      0 :::11315                :::*                    LISTEN      4294/scribed
tcp6       0      0 :::38422                :::*                    LISTEN      52781/sshd
tcp6       0      0 :::22                   :::*                    LISTEN      1305/sshd

自动配置 scribe.sh

#/bin/bash
#scribe server installation for CentOS 7

if [ "$#" -ne 1 ]
then
	echo "Usage: bash $0 project_name"
	exit 1
fi

yum -y install git make flex bison libtool automake openssl-devel libevent libevent-devel python-devel gcc-c++ byacc java-1.7.0-openjdk ant


#===============update autoconf===============
cd ~
sudo rpm -e --nodeps `rpm -qf /usr/bin/autoconf`
wget http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz
tar xzf autoconf-2.69.tar.gz
cd autoconf-2.69/
./configure
make
make install

echo "export PATH=$PATH:/usr/local/bin" >> /etc/bashrc
export PATH=$PATH:/usr/local/bin


#.===============install boost===============
wget http://jaist.dl.sourceforge.net/project/boost/boost/1.44.0/boost_1_44_0.tar.gz
tar xzf boost_1_44_0.tar.gz
cd boost_1_44_0/
./bootstrap.sh
./bjam install

#add new lib path for libboost
#echo "/usr/local/lib" >> /etc/ld.so.conf.d/localbin.conf
#ldconfig

#===============install thrift===============
cd ~
git clone https://github.com/apache/thrift.git
cd thrift/
git fetch
git branch -a
git checkout 0.7.x
./bootstrap.sh
./configure
mv compiler/cpp/thrifty.hh compiler/cpp/thrifty.h
make
make install
cd lib/py
python setup.py install

#===============install fb303===============
cd ~/thrift/contrib/fb303/
./bootstrap.sh
./configure CPPFLAGS="-DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H"
make
make install
cd py/
python setup.py install

#===============install libevent =============== 
cd ~
wget https://github.com/downloads/libevent/libevent/libevent-1.4.10-stable.tar.gz
tar xf libevent-1.4.10-stable.tar.gz 
cd libevent-1.4.10-stable
./configure
make
make install

#setup scribe server
mkdir -p /opt/scribe_server/$PROJECT/
cd /opt/scribe_server/$PROJECT/
cat <<EOF>>scribed.conf
port=12315
max_msg_per_second=2000000
check_interval=5
new_thread_per_category=yes
num_thrift_server_threads=5

<store>
category=default
type=buffer

target_write_size=20480
max_write_interval=1
buffer_send_rate=2
retry_interval=30
retry_interval_range=10

<primary>
type=file
fs_type=std
file_path=/data/scribe_primary/${PROJECT}
max_size=1000000000
rotate_period=daily
rotate_hour=0
</primary>

<secondary>
type=file
fs_type=std
file_path=/data/scribe_secondary/${PROJECT}
max_size=1000000000
rotate_period=daily
rotate_hour=0
</secondary>
</store>
EOF

cat <<EOF>>run_scribe.sh
#!/bin/bash

while true
do
	export LD_LIBRARY_PATH=/usr/lib:/usr/local/lib
	scribed scribed.conf 1>>output/scribe.log 2>>output/scribe.err
	sleep 2
done
EOF

chmod +x run_scribe.sh
mkdir output

echo "Done! Please modify config and iptables, and download scribed"

Docker 容器部署 Scribe

上述方案受到依赖环境库影响手动部署非常困难，推荐使用 docker 方式快速部署

polonaiz/facebook-scribe

Scribe is a server for aggregating log data streamed in real time from a large number of servers. (https://github.com/facebookarchive/scribe)

Dockerfile: https://github.com/polonaiz/facebook-scribe-docker/blob/master/Dockerfile

Usage: https://github.com/polonaiz/facebook-scribe-docker/blob/master/Makefile

Default Configuration: https://github.com/polonaiz/facebook-scribe-docker/blob/master/default.conf

基于 polonaiz/facebook-scribe 修改后的 docker image，配置参数和线上保持一致

# Makefile

all: build

build:
        docker build . --tag 'wangao/facebook-scribe'

clean:
        docker rmi -f 'wangao/facebook-scribe'

setup:
        sudo mkdir -p /data/gop/live/primary/gop
        sudo mkdir -p /data/gop/live/secondary/gop
        sudo chown -R ${USER}.${USER} /data/gop

stop:
        docker rm -f 'scribe'

start:
        docker run \
                --detach \
                --restart always \
                --name 'scribe' \
                --publish 11315:11315 \
                --mount type=bind,source=/data/gop/,destination=/data/gop/,consistency=consistent \
                -v /etc/localtime:/etc/localtime \
                wsgzao/facebook-scribe

test:
        docker exec -it scribe bash -c 'date | scribe_cat test; sleep 1'
        tail /data/gop/live/primary/test/test_current

push:
        docker login
        docker push wangao/facebook-scribe

# how to work
make setup
make start