Merge pull request #1090 from NixOS/munin-rework

munin: refactor package and add nixos service

nixos/modules/module-list.nix

@@ -130,6 +130,7 @@
   ./services/monitoring/dd-agent.nix
   ./services/monitoring/graphite.nix
   ./services/monitoring/monit.nix
+  ./services/monitoring/munin.nix
   ./services/monitoring/nagios/default.nix
   ./services/monitoring/smartd.nix
   ./services/monitoring/statsd.nix

nixos/modules/services/monitoring/munin.nix

@@ -0,0 +1,216 @@
+{ config, pkgs, ... }:
+
+# TODO: support munin-async
+# TODO: LWP/Pg perl libs aren't recognized
+
+# TODO: support fastcgi
+# http://munin-monitoring.org/wiki/CgiHowto2
+# spawn-fcgi -s /var/run/munin/fastcgi-graph.sock -U www-data   -u munin -g munin /usr/lib/munin/cgi/munin-cgi-graph
+# spawn-fcgi -s /var/run/munin/fastcgi-html.sock  -U www-data   -u munin -g munin /usr/lib/munin/cgi/munin-cgi-html
+# https://paste.sh/vofcctHP#-KbDSXVeWoifYncZmLfZzgum
+# nginx http://munin.readthedocs.org/en/latest/example/webserver/nginx.html
+
+
+with pkgs.lib;
+
+let
+  nodeCfg = config.services.munin-node;
+  cronCfg = config.services.munin-cron;
+
+  muninPlugins = pkgs.stdenv.mkDerivation {
+    name = "munin-available-plugins";
+    buildCommand = ''
+      mkdir -p $out
+
+      cp --preserve=mode ${pkgs.munin}/lib/plugins/* $out/
+
+      for file in $out/*; do
+        case "$file" in
+            plugin.sh) continue;;
+        esac
+
+        # read magic makers from the file
+        family=$(sed -nr 's/.*#%#\s+family\s*=\s*(\S+)\s*/\1/p' $file)
+        cap=$(sed -nr 's/.*#%#\s+capabilities\s*=\s*(.+)/\1/p' $file)
+
+        wrapProgram $file \
+          --set PATH "/run/current-system/sw/bin:/run/current-system/sw/sbin" \
+          --set MUNIN_LIBDIR "${pkgs.munin}/lib" \
+          --set MUNIN_PLUGSTATE "/var/run/munin"
+ 
+        # munin uses markers to tell munin-node-configure what a plugin can do
+        echo "#%# family=$family" >> $file
+        echo "#%# capabilities=$cap" >> $file
+      done
+
+      # NOTE: we disable disktstats because plugin seems to fail and it hangs html generation (100% CPU + memory leak)
+      rm -f $out/diskstats
+    '';
+    buildInputs = [ pkgs.makeWrapper ];
+  };
+
+  muninConf = pkgs.writeText "munin.conf"
+    ''
+      dbdir     /var/lib/munin
+      htmldir   /var/www/munin
+      logdir    /var/log/munin
+      rundir    /var/run/munin
+
+      ${cronCfg.extraGlobalConfig}
+      
+      ${cronCfg.hosts}
+    '';
+
+  nodeConf = pkgs.writeText "munin-node.conf"
+    ''
+      log_level 3
+      log_file Sys::Syslog
+      port 4949
+      host *
+      background 0
+      user root
+      group root
+      host_name ${config.networking.hostName}
+      setsid 0
+  
+      # wrapped plugins by makeWrapper being with dots
+      ignore_file ^\.
+      
+      allow ^127\.0\.0\.1$
+
+      ${nodeCfg.extraConfig}
+    '';
+in
+
+{
+
+  options = {
+
+    services.munin-node = {
+
+      enable = mkOption {
+        default = false;
+        description = ''
+          Enable Munin Node agent. Munin node listens on 0.0.0.0 and
+          by default accepts connections only from 127.0.0.1 for security reasons.
+
+          See <link xlink:href='http://munin-monitoring.org/wiki/munin-node' />.
+        '';
+      };
+      
+      extraConfig = mkOption {
+        default = "";
+        description = ''
+          <filename>munin-node.conf</filename> extra configuration. See
+          <link xlink:href='http://munin-monitoring.org/wiki/munin-node.conf' />
+        '';
+      };
+
+      # TODO: add option to add additional plugins
+
+    };
+
+    services.munin-cron = {
+
+      enable = mkOption {
+        default = false;
+        description = ''
+          Enable munin-cron. Takes care of all heavy lifting to collect data from
+          nodes and draws graphs to html. Runs munin-update, munin-limits,
+          munin-graphs and munin-html in that order.
+ 
+          HTML output is in <filename>/var/www/munin/</filename>, configure your
+          favourite webserver to serve static files.
+        '';
+        example = literalExample ''
+          services = {
+             munin-node.enable = true;
+             munin-cron = {
+               enable = true;
+               hosts = '''
+                 [''${config.networking.hostName}]
+                 address localhost
+               ''';
+               extraGlobalConfig = '''
+                 contact.email.command mail -s "Munin notification for ''${var:host}" someone@example.com
+               ''';
+             };
+          };
+        '';
+      };
+      
+      extraGlobalConfig = mkOption {
+        default = "";
+        description = ''
+          <filename>munin.conf</filename> extra global configuration.
+          See <link xlink:href='http://munin-monitoring.org/wiki/munin-node' />.
+          Useful to setup notifications, see
+          <link xlink:href='http://munin-monitoring.org/wiki/HowToContact' />
+        '';
+      };
+
+      hosts = mkOption {
+        example = ''
+          [''${config.networking.hostName}]
+          address localhost
+        '';
+        description = ''
+          Definitions of hosts of nodes to collect data from. Needs at least one
+          hosts for cron to succeed. See
+          <link xlink:href='http://munin-monitoring.org/wiki/munin.conf' />
+        '';
+      };
+ 
+    };
+
+  };
+
+  config = mkMerge [ (mkIf (nodeCfg.enable || cronCfg.enable)  {
+
+    environment.systemPackages = [ pkgs.munin ];
+
+    users.extraUsers = [{
+      name = "munin";
+      description = "Munin monitoring user";
+      group = "munin";
+    }];
+
+    users.extraGroups = [{
+      name = "munin";
+    }];
+
+  }) (mkIf nodeCfg.enable {
+
+    systemd.services.munin-node = {
+      description = "Munin node, the agent process";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      path = [ pkgs.munin ];
+      environment.MUNIN_PLUGSTATE = "/var/run/munin";
+      serviceConfig = {
+        ExecStart = "${pkgs.munin}/sbin/munin-node --config ${nodeConf} --servicedir /etc/munin/plugins/";
+      };
+    };
+
+    system.activationScripts.munin-node = ''
+      echo "updating munin plugins..."
+
+      export PATH="/run/current-system/sw/bin:/run/current-system/sw/sbin";
+      mkdir -p /etc/munin/plugins
+      rm -rf /etc/munin/plugins/*
+      ${pkgs.munin}/sbin/munin-node-configure --shell --families contrib,auto,manual --config ${nodeConf} --libdir=${muninPlugins} --servicedir=/etc/munin/plugins 2>/dev/null | ${pkgs.bash}/bin/bash
+    '';
+
+  }) (mkIf cronCfg.enable {
+
+    services.cron.systemCronJobs = [
+      "*/5 * * * * munin ${pkgs.munin}/bin/munin-cron --config ${muninConf}"
+    ];
+
+    system.activationScripts.munin-cron = stringAfter [ "users" "groups" ] ''
+      mkdir -p /var/{run,log,www,lib}/munin
+      chown -R munin:munin /var/{run,log,www,lib}/munin
+    '';
+
+  })];
+}

pkgs/servers/monitoring/munin/adding_servicedir_munin-node.patch

@@ -0,0 +1,84 @@
+From 75a3ec48814e7b9a9b22259a04009076363be3f1 Mon Sep 17 00:00:00 2001
+From: Igor Kolar <igor.kolar@gmail.com>
+Date: Thu, 17 Oct 2013 00:48:23 +0200
+Subject: [PATCH 1/2] node: added --servicedir switch to munin-node
+
+This code is copied over from munin-node-config, that already does the same
+---
+ node/sbin/munin-node | 7 ++++++-
+ 1 file changed, 6 insertions(+), 1 deletion(-)
+
+diff --git a/node/sbin/munin-node b/node/sbin/munin-node
+index 7b2e180..0a93450 100755
+--- a/node/sbin/munin-node
++++ b/node/sbin/munin-node
+@@ -35,7 +35,7 @@ use Munin::Node::OS;
+ use Munin::Node::Service;
+ use Munin::Node::Server;
+ 
+-my $servicedir;
++my $servicedir = "$Munin::Common::Defaults::MUNIN_CONFDIR/plugins";
+ my $sconfdir = "$Munin::Common::Defaults::MUNIN_CONFDIR/plugin-conf.d";
+ my $conffile = "$Munin::Common::Defaults::MUNIN_CONFDIR/munin-node.conf";
+ my $DEBUG    = 0;
+@@ -101,6 +101,7 @@ sub parse_args
+ 
+     print_usage_and_exit() unless GetOptions(
+         "config=s"     => \$conffile,
++        "servicedir=s" => \$servicedir,
+         "debug!"       => \$DEBUG,
+         "pidebug!"     => \$PIDEBUG,
+         "paranoia!"    => \$paranoia,
+@@ -166,6 +167,10 @@ and returning the output they produce.
+ 
+ Use E<lt>fileE<gt> as configuration file. [@@CONFDIR@@/munin-node.conf]
+ 
++=item B<< --servicedir <dir> >>
++
++Override plugin directory [@@CONFDIR@@/plugins/]
++
+ =item B< --[no]paranoia >
+ 
+ Only run plugins owned by root. Check permissions as well. [--noparanoia]
+-- 
+1.8.4
+
+
+From b8e17cbe73ae4c71b93ff5687ba86db1d0c1f5bd Mon Sep 17 00:00:00 2001
+From: Steve Schnepp <steve.schnepp@pwkf.org>
+Date: Thu, 17 Oct 2013 11:52:10 +0200
+Subject: [PATCH 2/2] node: untaint the service-dir args
+
+---
+ node/sbin/munin-node | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/node/sbin/munin-node b/node/sbin/munin-node
+index 0a93450..909c8c4 100755
+--- a/node/sbin/munin-node
++++ b/node/sbin/munin-node
+@@ -99,9 +99,10 @@ sub parse_args
+ {
+     my @ORIG_ARGV  = @ARGV;
+ 
++    my $servicedir_cmdline;
+     print_usage_and_exit() unless GetOptions(
+         "config=s"     => \$conffile,
+-        "servicedir=s" => \$servicedir,
++        "servicedir=s" => \$servicedir_cmdline,
+         "debug!"       => \$DEBUG,
+         "pidebug!"     => \$PIDEBUG,
+         "paranoia!"    => \$paranoia,
+@@ -109,6 +110,9 @@ sub parse_args
+         "help"         => \&print_usage_and_exit,
+     );
+ 
++    # We untaint the args brutally, since the sysadm should know what he does
++    $servicedir = $1 if defined $servicedir_cmdline && $servicedir_cmdline =~ m/(.*)/;
++
+     # Reset ARGV (for HUPing)
+     @ARGV = @ORIG_ARGV;
+ 
+-- 
+1.8.4
+

pkgs/servers/monitoring/munin/default.nix

@@ -1,9 +1,5 @@
 { stdenv, fetchurl, makeWrapper, which, coreutils, rrdtool, perl, perlPackages
-, python, ruby, openjdk }:
+, python, ruby, openjdk, nettools }:
-
-# TODO: split into server/node derivations
-
-# FIXME: munin tries to write log files and web graphs to its installation path.
 
 stdenv.mkDerivation rec {
   version = "2.0.17";
@@ -19,6 +15,7 @@ stdenv.mkDerivation rec {
     which
     coreutils
     rrdtool
+    nettools
     perl
     perlPackages.ModuleBuild
     perlPackages.HTMLTemplate
@@ -36,17 +33,49 @@ stdenv.mkDerivation rec {
     perlPackages.NetServer
     perlPackages.ListMoreUtils
     perlPackages.TimeHiRes
+    perlPackages.LWPUserAgent
+    perlPackages.DBDPg
     python
     ruby
     openjdk
+    # tests
+    perlPackages.TestLongString
+    perlPackages.TestDifferences
+    perlPackages.TestDeep
+    perlPackages.TestMockModule
+    perlPackages.TestMockObject
+    perlPackages.FileSlurp
+    perlPackages.IOStringy
+  ];
+
+  # TODO: tests are failing http://munin-monitoring.org/ticket/1390#comment:1
+  # NOTE: important, test command always exits with 0, think of a way to abort the build once tests pass
+  doCheck = false;
+
+  checkPhase = ''
+   export PERL5LIB="$PERL5LIB:${rrdtool}/lib/perl"
+   LC_ALL=C make -j1 test 
+  '';
+
+  patches = [
+    # https://rt.cpan.org/Public/Bug/Display.html?id=75112
+    ./dont_preserve_source_dir_permissions.patch
+
+    # https://github.com/munin-monitoring/munin/pull/134
+    ./adding_servicedir_munin-node.patch
   ];
 
   preBuild = ''
+    substituteInPlace "Makefile" \
+      --replace "/bin/pwd" "pwd"
+
+    # munin checks at build time if user/group exists, unpure
     sed -i '/CHECKUSER/d' Makefile
     sed -i '/CHOWN/d' Makefile
     sed -i '/CHECKGROUP/d' Makefile
-    substituteInPlace "Makefile" \
+
-      --replace "/usr/pwd" "pwd"
+    # munin hardcodes PATH, we need it to obey $PATH
+    sed -i '/ENV{PATH}/d' node/lib/Munin/Node/Service.pm
   '';
 
   # DESTDIR shouldn't be needed (and shouldn't have worked), but munin
@@ -60,7 +89,7 @@ stdenv.mkDerivation rec {
     PYTHON=${python}/bin/python
     RUBY=${ruby}/bin/ruby
     JAVARUN=${openjdk}/bin/java
-    HOSTNAME=default
+    PLUGINUSER=munin
   '';
 
   postFixup = ''
@@ -78,7 +107,8 @@ stdenv.mkDerivation rec {
         case "$file" in
             *.jar) continue;;
         esac
-        wrapProgram "$file" --set PERL5LIB $out/lib/perl5/site_perl:${perlPackages.Log4Perl}/lib/perl5/site_perl:${perlPackages.IOSocketInet6}/lib/perl5/site_perl:${perlPackages.Socket6}/lib/perl5/site_perl:${perlPackages.URI}/lib/perl5/site_perl:${perlPackages.DBFile}/lib/perl5/site_perl:${perlPackages.DateManip}/lib/perl5/site_perl:${perlPackages.HTMLTemplate}/lib/perl5/site_perl:${perlPackages.FileCopyRecursive}/lib/perl5/site_perl:${perlPackages.FCGI}/lib/perl5/site_perl:${perlPackages.NetSNMP}/lib/perl5/site_perl:${perlPackages.NetServer}/lib/perl5/site_perl:${perlPackages.ListMoreUtils}/lib/perl5/site_perl:${perlPackages.TimeHiRes}/lib/perl5/site_perl:${rrdtool}/lib/perl
+        wrapProgram "$file" \
+          --set PERL5LIB "$out/lib/perl5/site_perl:${perlPackages.Log4Perl}/lib/perl5/site_perl:${perlPackages.IOSocketInet6}/lib/perl5/site_perl:${perlPackages.Socket6}/lib/perl5/site_perl:${perlPackages.URI}/lib/perl5/site_perl:${perlPackages.DBFile}/lib/perl5/site_perl:${perlPackages.DateManip}/lib/perl5/site_perl:${perlPackages.HTMLTemplate}/lib/perl5/site_perl:${perlPackages.FileCopyRecursive}/lib/perl5/site_perl:${perlPackages.FCGI}/lib/perl5/site_perl:${perlPackages.NetSNMP}/lib/perl5/site_perl:${perlPackages.NetServer}/lib/perl5/site_perl:${perlPackages.ListMoreUtils}/lib/perl5/site_perl:${perlPackages.TimeHiRes}/lib/perl5/site_perl:${rrdtool}/lib/perl:${perlPackages.DBDPg}/lib/perl5/site_perl:${perlPackages.LWPUserAgent}/lib/perl5/site_perl"
     done
   '';
 

pkgs/servers/monitoring/munin/dont_preserve_source_dir_permissions.patch

@@ -0,0 +1,18 @@
+# https://rt.cpan.org/Public/Bug/Display.html?id=75112
+diff --git a/master/lib/Munin/Master/HTMLOld.pm b/master/lib/Munin/Master/HTMLOld.pm
+index 2b6e71f..c0aa2c0 100644
+--- a/master/lib/Munin/Master/HTMLOld.pm
++++ b/master/lib/Munin/Master/HTMLOld.pm
+@@ -711,10 +711,12 @@ sub emit_main_index {
+ 
+ sub copy_web_resources {
+     my ($staticdir, $htmldir) = @_;
++       local $File::Copy::Recursive::KeepMode = 0;
+ 	unless(dircopy($staticdir, "$htmldir/static")){
+ 		ERROR "[ERROR] Could not copy contents from $staticdir to $htmldir";
+ 		die "[ERROR] Could not copy contents from $staticdir to $htmldir";
+ 	}
++       local $File::Copy::Recursive::KeepMode = 1;
+ }
+ 
+ sub instanciate_comparison_templates {