Metrics visualisation and collection with Graphite, Grafana and python

Metrics visualisation and collection with Graphite, Grafana and python In this post I'll demonstrate how to collect various metrics using a simple python daemon, send them to Graphite and create dashboards with Grafana that are stored in Elasticsearch.

For more detailed explanation of what Graphite is and it's Carbon and Whisper components please read my other blog post here.

First lets install and configure Graphite:

[root@server1 ~] apt-get install -y python2.6 python-pip python-cairo python-django python-django-tagging [root@server1 ~] apt-get install -y libapache2-mod-wsgi python-twisted python-memcache python-pysqlite2 python-simplejson

[root@server1 ~] pip install whisper [root@server1 ~] pip install carbon [root@server1 ~] pip install graphite-web

[root@server1 ~] cat /etc/apache2/sites-available/graphite

WSGISocketPrefix /etc/httpd/wsgi/

<VirtualHost *:80>

Header set Access-Control-Allow-Origin "*" Header set Access-Control-Allow-Methods "GET, OPTIONS" Header set Access-Control-Allow-Headers "origin, authorization, accept"

ServerName graphite.prod.lon3.example.net DocumentRoot "/opt/graphite/webapp" ErrorLog /opt/graphite/storage/log/webapp/error.log CustomLog /opt/graphite/storage/log/webapp/access.log common

WSGIDaemonProcess graphite processes=5 threads=5 display-name='%{GROUP}' inactivity-timeout=120 WSGIProcessGroup graphite WSGIApplicationGroup %{GLOBAL} WSGIImportScript /opt/graphite/conf/graphite.wsgi process-group=graphite application-group=%{GLOBAL}

WSGIScriptAlias / /opt/graphite/conf/graphite.wsgi

Alias /content/ /opt/graphite/webapp/content/ <Location "/content/"> SetHandler None </Location>

Alias /media/ "@DJANGO_ROOT@/contrib/admin/media/" <Location "/media/"> SetHandler None </Location>

<Directory /opt/graphite/conf/> Order deny,allow Allow from all </Directory>

</VirtualHost>

[root@server1 ~] mkdir -p /etc/httpd/wsgi/ [root@server1 ~] cd /opt/graphite/conf/ [root@server1 /opt/graphite/conf] cat carbon.conf

[cache] ENABLE_LOGROTATION = True USER = MAX_CACHE_SIZE = inf MAX_UPDATES_PER_SECOND = 500 MAX_CREATES_PER_MINUTE = 50 LINE_RECEIVER_INTERFACE = 0.0.0.0 LINE_RECEIVER_PORT = 2003 ENABLE_UDP_LISTENER = False UDP_RECEIVER_INTERFACE = 0.0.0.0 UDP_RECEIVER_PORT = 2003 PICKLE_RECEIVER_INTERFACE = 0.0.0.0 PICKLE_RECEIVER_PORT = 2004 LOG_LISTENER_CONNECTIONS = True USE_INSECURE_UNPICKLER = False CACHE_QUERY_INTERFACE = 0.0.0.0 CACHE_QUERY_PORT = 7002 USE_FLOW_CONTROL = True LOG_UPDATES = False LOG_CACHE_HITS = False LOG_CACHE_QUEUE_SORTS = True CACHE_WRITE_STRATEGY = sorted WHISPER_AUTOFLUSH = False WHISPER_FALLOCATE_CREATE = True [relay] LINE_RECEIVER_INTERFACE = 0.0.0.0 LINE_RECEIVER_PORT = 2013 PICKLE_RECEIVER_INTERFACE = 0.0.0.0 PICKLE_RECEIVER_PORT = 2014 LOG_LISTENER_CONNECTIONS = True RELAY_METHOD = rules REPLICATION_FACTOR = 1 DESTINATIONS = 127.0.0.1:2004 MAX_DATAPOINTS_PER_MESSAGE = 500 MAX_QUEUE_SIZE = 10000 USE_FLOW_CONTROL = True [aggregator] LINE_RECEIVER_INTERFACE = 0.0.0.0 LINE_RECEIVER_PORT = 2023 PICKLE_RECEIVER_INTERFACE = 0.0.0.0 PICKLE_RECEIVER_PORT = 2024 LOG_LISTENER_CONNECTIONS = True FORWARD_ALL = True DESTINATIONS = 127.0.0.1:2004 REPLICATION_FACTOR = 1 MAX_QUEUE_SIZE = 10000 USE_FLOW_CONTROL = True MAX_DATAPOINTS_PER_MESSAGE = 500 MAX_AGGREGATION_INTERVALS = 5

[root@server1 /opt/graphite/conf] cp graphite.wsgi.example graphite.wsgi [root@server1 /opt/graphite/conf] cat storage-schemas.conf

[carbon] pattern = ^carbon\. retentions = 300:90d

[default_5min_for_90day] pattern = .* retentions = 300s:90d

[root@server1 /opt/graphite/conf] cat storage-aggregation.conf

[min] pattern = \.min$ xFilesFactor = 0.1 aggregationMethod = min

[max] pattern = \.max$ xFilesFactor = 0.1 aggregationMethod = max

[sum] pattern = \.count$ xFilesFactor = 0 aggregationMethod = sum

[default_average] pattern = .* xFilesFactor = 0.5 aggregationMethod = average

[root@server1 /opt/graphite/conf] cd /opt/graphite/webapp/graphite [root@server1 /opt/graphite/conf] python manage.py syncdb [root@server1 /opt/graphite/conf] chown -R www-data:www-data /opt/graphite/storage/ [root@server1 /opt/graphite/conf] a2ensite graphite [root@server1 /opt/graphite/conf] a2dissite default [root@server1 /opt/graphite/conf] a2enmod headers [root@server1 /opt/graphite/conf] /etc/init.d/apache2 restart [root@server1 /opt/graphite/conf] /opt/graphite/bin/carbon-cache.py start