added dblog/93_DbLog.pm and samples in contrib directory, commandref.html update

git-svn-id: https://svn.fhem.de/fhem/trunk@140 2b470e98-0d58-463d-a4d8-8e2adae1ed80

fhem/CHANGED

@@ -373,6 +373,7 @@
   - feature: defattr renamed to setdefaultattr (Rudi, 29.12)
   - feature: device spec (list/range/regexp) for most commands implemented
   - feature: %NAME, %EVENT, %TYPE parameters in notify definition
+  - feature: added 93_DbLog.pm, database logging facility (Boris, 30.12.)
 
 - TODO
   emem -2.5kW / getDevData for emwz -1

fhem/HISTORY

@@ -176,3 +176,6 @@
 - Boris Sat Dec 29 16:56:00 CET 2007
   - %NAME, %EVENT, %TYPE parameters in notify definition, commandref.html update
 
+- Boris Sun Dec 30 22:35:00 CET 2007
+  - added dblog/93_DbLog.pm and samples in contrib directory, commandref.html update
+

fhem/contrib/README

@@ -7,6 +7,7 @@
   Support for FS10. Read README.FS10, you have to install pcwsd first.
 - 91_DbLog.pm
   Example to log data in a (DBI supported) database (MySQL, Oracle, etc)
+  see dblog for a full-featured database log
 - 99_ALARM.pm
   Example for a Low Budget ALARM System by Martin
 - 99_SUNRISE.pm
@@ -43,3 +44,5 @@
   Peter's RRD support. See the HOWTO
 - serial.pm
   Serial line analyzer
+- dblog
+  Support for a full-featured database log by Boris. See the README.

fhem/contrib/dblog/93_DbLog.pm

@@ -0,0 +1,262 @@
+##############################################
+#
+# 93_DbLog.pm
+# written by Dr. Boris Neubert 2007-12-30
+# e-mail: omega at online dot de
+#
+##############################################
+
+package main;
+use strict;
+use warnings;
+use DBI;
+
+sub DbLog($$$);
+
+
+################################################################
+sub
+DbLog_Initialize($)
+{
+  my ($hash) = @_;
+
+  $hash->{DefFn} = "DbLog_Define";
+  $hash->{UndefFn} = "DbLog_Undef";
+  $hash->{NotifyFn} = "DbLog_Log";
+  $hash->{AttrFn}   = "DbLog_Attr";
+  $hash->{AttrList} = "disable:0,1";
+}
+
+#####################################
+sub
+DbLog_Define($@)
+{
+  my ($hash, $def) = @_;
+  my @a = split("[ \t][ \t]*", $def);
+
+  return "wrong syntax: define <name> DbLog configuration regexp" 
+	if(int(@a) != 4);
+
+  my $regexp    	= $a[3];
+
+  eval { "Hallo" =~ m/^$regexp$/ };
+  return "Bad regexp: $@" if($@);
+  $hash->{REGEXP} = $regexp;
+
+  $hash->{configuration}= $a[2];
+
+  return "Can't connect to database." if(!DbLog_Connect($hash));
+
+  $hash->{STATE} = "active";
+
+  return undef;
+}
+
+#####################################
+sub
+DbLog_Undef($$)
+{
+  my ($hash, $name) = @_;
+  my $dbh= $hash->{DBH};
+  $dbh->disconnect() if(defined($dbh)); 
+  return undef;
+}
+
+################################################################
+sub
+DbLog_Attr(@)
+{
+  my @a = @_;
+  my $do = 0;
+
+  if($a[0] eq "set" && $a[2] eq "disable") {
+    $do = (!defined($a[3]) || $a[3]) ? 1 : 2;
+  }
+  $do = 2 if($a[0] eq "del" && (!$a[2] || $a[2] eq "disable"));
+  return if(!$do);
+
+  $defs{$a[1]}{STATE} = ($do == 1 ? "disabled" : "active");
+
+  return undef;
+}
+
+################################################################
+sub
+DbLog_ParseEvent($$)
+{
+  my ($type, $event)= @_;
+  my @result;
+
+  # split the event into reading and argument
+  # "day-temp: 22.0 (Celsius)" -> "day-temp", "22.0 (Celsius)"
+  my @parts= split(/: /,$event);
+  my $reading= $parts[0]; if(!defined($reading)) { $reading= ""; }
+  my $arg= $parts[1];
+
+  # the interpretation of the argument depends on the device type
+
+  #default
+  my $value= $arg; if(!defined($value)) { $value= ""; }
+  my $unit= "";
+
+
+  # EMEM, M232Counter, M232Voltage return plain numbers
+  if(($type eq "M232Voltage") || 
+     ($type eq "M232Counter") ||
+     ($type eq "EMEM")) {
+  }
+  # FHT 
+  elsif($type eq "FHT") {
+     if($reading =~ m(-temp)) { $value=~ s/ \(Celsius\)//; $unit= "°C"; }
+     if($reading =~ m(temp-offset)) { $value=~ s/ \(Celsius\)//; $unit= "°C"; }
+     if($reading eq "actuator") { $value=~ s/%//; $value= $value*1.; $unit= "%"; }
+  }
+  # KS300
+  elsif($type eq "KS300") {
+     if($event =~ m(T:.*)) { $reading= "data"; $value= $event; }
+     if($event =~ m(avg_day)) { $reading= "data"; $value= $event; }
+     if($event =~ m(avg_month)) { $reading= "data"; $value= $event; }
+     if($reading eq "temperature") { $value=~ s/ \(Celsius\)//; $unit= "°C"; }
+     if($reading eq "wind") { $value=~ s/ \(km\/h\)//; $unit= "km/h"; }
+     if($reading eq "rain") { $value=~ s/ \(l\/m2\)//; $unit= "l/m2"; }
+     if($reading eq "rain_raw") { $value=~ s/ \(counter\)//; $unit= ""; }
+     if($reading eq "humidity") { $value=~ s/ \(\%\)//; $unit= "%"; }
+     if($reading eq "israining") { 
+	$value=~ s/ \(yes\/no\)//; 
+        $value=~ s/no/0/;
+        $value=~ s/yes/1/;
+      }
+  }
+  # HMS
+  elsif($type eq "HMS") {
+     if($event =~ m(T:.*)) { $reading= "data"; $value= $event; }
+     if($reading eq "temperature") { $value=~ s/ \(Celsius\)//; $unit= "°C"; }
+     if($reading eq "humidity") { $value=~ s/ \(\%\)//; $unit= "%"; }
+     if($reading eq "battery") { 
+        $value=~ s/ok/1/;
+        $value=~ s/replaced/1/;
+        $value=~ s/empty/0/;
+     }
+   }
+  
+
+  @result= ($reading,$value,$unit);
+  return @result;
+}
+
+
+################################################################
+sub
+DbLog_Log($$)
+{
+  # Log is my entry, Dev is the entry of the changed device
+  my ($log, $dev) = @_;
+ 
+  # name and type required for parsing
+  my $n= $dev->{NAME};
+  my $t= $dev->{TYPE};
+
+  # timestamp in SQL format YYYY-MM-DD hh:mm:ss
+  #my ($sec,$min,$hr,$day,$mon,$yr,$wday,$yday,$isdst)= localtime(time);
+  #my $ts= sprintf("%04d-%02d-%02d %02d:%02d:%02d", $yr+1900,$mon+1,$day,$hr,$min,$sec);
+
+  my $re = $log->{REGEXP};
+  my $max = int(@{$dev->{CHANGED}});
+  for (my $i = 0; $i < $max; $i++) {
+    my $s = $dev->{CHANGED}[$i];
+    $s = "" if(!defined($s));
+    if($n =~ m/^$re$/ || "$n:$s" =~ m/^$re$/) {
+      my $ts = TimeNow();
+      $ts = $dev->{CHANGETIME}[$i] if(defined($dev->{CHANGETIME}[$i]));
+      # $ts is in SQL format YYYY-MM-DD hh:mm:ss
+
+      my @r= DbLog_ParseEvent($t, $s);
+      my $reading= $r[0];
+      my $value= $r[1];
+      my $unit= $r[2];
+
+      my $is= "(TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES " .
+         "('$ts', '$n', '$t', '$s', '$reading', '$value', '$unit')";
+      DbLog_ExecSQL($log, "INSERT INTO history" . $is);
+      DbLog_ExecSQL($log, "DELETE FROM current WHERE (DEVICE='$n') AND (READING='$reading')");
+      DbLog_ExecSQL($log, "INSERT INTO current" . $is);
+
+
+    }
+  }
+
+  return "";
+}
+
+
+################################################################
+sub
+DbLog_Connect($)
+{
+  my ($hash)= @_;
+
+  my $configfilename= $hash->{configuration};
+  if(!open(CONFIG, $configfilename)) {
+	Log 1, "Cannot open database configuration file $configfilename.";
+	return 0; }
+  my @config=<CONFIG>;
+  close(CONFIG);
+
+  my %dbconfig;
+  eval join("", @config);
+
+  my $dbconn= $dbconfig{connection};
+  my $dbuser= $dbconfig{user};
+  my $dbpassword= $dbconfig{password};
+
+  Log 3, "Connecting to database $dbconn with user $dbuser";
+  my $dbh = DBI->connect_cached("dbi:$dbconn", $dbuser, $dbpassword);
+  if(!$dbh) {
+    Log 1, "Can't connect to $dbconn: $DBI::errstr";
+    return 0;
+  }
+  Log 3, "Connection to db $dbconn established";
+  $hash->{DBH}= $dbh;
+  return 1;
+}
+
+################################################################
+sub
+DbLog_ExecSQL1($$)
+{
+  my ($dbh,$sql)= @_;
+ 
+  my $sth = $dbh->do($sql);
+  if(!$sth) {
+    Log 2, "DBLog error: " . $DBI::errstr;
+    return 0;
+  }
+  return 1;
+}
+
+sub
+DbLog_ExecSQL($$)
+{
+  my ($hash,$sql)= @_;
+ 
+  Log 5, "Executing $sql";
+  my $dbh= $hash->{DBH};
+  if(!DbLog_ExecSQL1($dbh,$sql)) {
+    #retry
+    $dbh->disconnect();
+    if(!DbLog_Connect($hash)) {
+      Log 2, "DBLog reconnect failed.";
+      return 0;
+    }
+    $dbh= $hash->{DBH};
+    if(!DbLog_ExecSQL1($dbh,$sql)) {
+      Log 2, "DBLog retry failed.";
+      return 0;
+    }
+    Log 2, "DBLog retry ok.";
+  }
+  return 1;
+}
+
+################################################################
+1;

fhem/contrib/dblog/README

@@ -0,0 +1,13 @@
+For usage instruction see commandref.html, section define
+2007-12-30bn
+
+- 93_DbLog.pm
+  copy this file into <modpath>/FHEM
+- db.conf
+  sample database configuration file
+- fhemdb_create.sql
+  sample sql command to create a mysql database for logging purposes
+- fhemdb_get.pl
+  sample perl script for retrieving the current (latest) data from
+  the logging database
+

fhem/contrib/dblog/db.conf

@@ -0,0 +1,11 @@
+#
+# database configuration file
+# 
+#
+#
+%dbconfig= (
+	connection => "mysql:database=fhem;host=db;port=3306",
+	user => "fhemuser",
+	password => "fhempassword",
+);
+

fhem/contrib/dblog/fhemdb_create.sql

@@ -0,0 +1,7 @@
+CREATE DATABASE `fhem` DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
+CREATE USER 'fhemuser'@'%' IDENTIFIED BY 'fhempassword';
+CREATE TABLE history (TIMESTAMP TIMESTAMP, DEVICE varchar(32), TYPE varchar(32), EVENT varchar(64), READING varchar(32), VALUE varchar(32), UNIT varchar(32));
+CREATE TABLE current (TIMESTAMP TIMESTAMP, DEVICE varchar(32), TYPE varchar(32), EVENT varchar(64), READING varchar(32), VALUE varchar(32), UNIT varchar(32));
+GRANT SELECT, INSERT, DELETE ON `fhem` .* TO 'fhemuser'@'%';
+
+

fhem/contrib/dblog/fhemdb_get.pl

@@ -0,0 +1,93 @@
+#!/usr/bin/perl
+#
+################################################################
+#
+#  Copyright notice
+#
+#  (c) 2007 Copyright: Dr. Boris Neubert (omega at online dot de)
+#  All rights reserved
+#
+#  This script free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  The GNU General Public License can be found at
+#  http://www.gnu.org/copyleft/gpl.html.
+#  A copy is found in the textfile GPL.txt and important notices to the license
+#  from the author is found in LICENSE.txt distributed with these scripts.
+#
+#  This script is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  This copyright notice MUST APPEAR in all copies of the script!
+#
+################################################################
+
+
+# 
+# this script returns the current reading for a device stored in
+# the fhem logging database
+#
+
+# Usage: 
+#   fhemdb_get.pl <device> <reading> [<reading> ...]
+# Example:
+#   fhemdb_get.pl ext.ks300 temperature humidity
+#
+#
+
+#
+# global configuration
+#
+my $dbconn      = "mysql:database=fhem;host=db;port=3306";
+my $dbuser      = "fhemuser";
+my $dbpassword  = "fhempassword";
+
+#
+# nothing to change below this line
+#
+
+use strict;
+use warnings;
+use DBI;
+
+(@ARGV>=2) || die "Usage: fhemdb_get.pl <device> <reading> [<reading> ... ]";
+
+my $device= $ARGV[0];
+my @readings=@ARGV; shift @readings;
+my $set= join(",", map({"\'" . $_ . "\'"} @readings));
+
+my $dbh= DBI->connect_cached("dbi:$dbconn", $dbuser, $dbpassword) || 
+	die "Cannot connect to $dbconn: $DBI::errstr";
+my $stm= "SELECT READING, VALUE FROM current WHERE
+	(DEVICE='$device') AND
+	(READING IN ($set))"; 
+my $sth= $dbh->prepare($stm) || 
+	die "Cannot prepare statement $stm: $DBI::errstr";
+my $rc= $sth->execute() ||
+	die "Cannot execute statement $stm: $DBI::errstr";
+
+my %rs;
+my $reading;
+my $value;
+while( ($reading,$value)= $sth->fetchrow_array) {
+	$rs{$reading}= $value;
+}
+foreach $reading (@readings) {
+	$value= $rs{$reading};
+	$value= "NULL" if(!defined($value));
+	print "$reading:$value ";
+}
+print "\n";
+die $sth->errstr if $sth->err;
+$dbh->disconnect();
+
+
+
+
+
+
+

fhem/docs/commandref.html

@@ -906,6 +906,60 @@ split in multiple lines<br><br>
     <br>
   </ul>
 
+  <a name="DbLog"></a>
+  <h4>Type DbLog</h4>
+  <ul>
+    <code>define &lt;name&gt; DbLog &lt;configfilename&gt; &lt;regexp&gt;</code>
+    <br><br>
+  
+    Log events to a database. The database connection is defined in
+    <code>&lt;configfilename&gt;</code> (see sample configuration file 
+    <code>db.conf</code>). The configuration is stored in a separate file
+    to avoid storing the password in the main configuration file and to have it
+    visible in the output of the <a href="#list">list</a> command.
+    <br><br>
+
+    You must have <code>93_DbLog.pm</code> in the <code>FHEM</code> subdirectory 
+    to make this work. Additionally, the modules <code>DBI</code> and 
+    <code>DBD::&lt;dbtype&gt;</code> need to be installed (use 
+    <code>cpan -i &lt;module&gt;</code> if your distribution does not have it).
+    <br><br>
+    <code>&lt;regexp&gt;</code> is the same as in <a href="#FileLog">FileLog</a>.
+    <br><br>
+    Sample code to create a MySQL database is in <code>fhemdb_create.sql</code>.
+    The database contains two tables: <code>current</code> and 
+    <code>history</code>. The latter contains all events whereas the former only
+    contains the last event for any given reading and device.
+    The columns have the following meaning:
+    <ol>
+	<li>TIMESTAMP: timestamp of event, e.g. <code>2007-12-30 21:45:22</code></li>
+	<li>DEVICE: device name, e.g. <code>Wetterstation</code></li>
+	<li>TYPE: device type, e.g. <code>KS300</code></li>
+	<li>EVENT: event specification as full string, 
+                                    	e.g. <code>humidity: 71 (%)</code></li>
+	<li>READING: name of reading extracted from event, 
+					e.g. <code>humidity</code></li>
+	<li>VALUE: actual reading extracted from event,
+					e.g. <code>71</code></li>
+	<li>UNIT: unit extracted from event, e.g. <code>%</code></li>
+    </ol>
+    The content of VALUE is optimized for automated post-processing, e.g.
+    <code>yes</code> is translated to <code>1</code>.
+    <br><br>
+    The current values can be retrieved by means of the perl script
+    <code>fhemdb_get.pl</code>. Its output is adapted to what a 
+    <a href="www.cacti.net">Cacti</a> data input method expects. 
+    Call <code>fhemdb_get.pl</code> without parameters to see the usage
+    information.
+    <br><br>
+    Examples:
+    <ul>
+    	<code># log everything to database</code><br>
+    	<code>define logdb DbLog /etc/fhem/db.conf .*:.*</code>
+    </ul>
+  </ul>
+
+
   <a name="at"></a>
   <h4>Type at</h4>
   <ul>
@@ -1010,7 +1064,7 @@ split in multiple lines<br><br>
       <code>define b3lampV3 notify btn3 "/usr/local/bin/setlamp "%""</code><br>
       <code>define b3lampV3 notify btn3 set lamp1 %;;set lamp2 %</code><br>
       <code>define wzMessLg notify wz:measured.* "/usr/local/bin/logfht @ "%""</code><br>
-      <code>define LogHToDB notify .*H:.* {DbLog("@","%")}</code><br>
+      <!-- <code>define LogHToDB notify .*H:.* {DbLog("@","%")}</code><br> -->
       <code>define LogUndef notify UNDEFINED "send-me-mail.sh "%""</code><br>
     </ul>
     <br>
@@ -1043,8 +1097,10 @@ split in multiple lines<br><br>
       href="#list">list</a> output in paranthesis after the device name, or the
       string you see when you do a detailed list of the device.</li>
 
-      <li>To use database logging, copy the file contrib/91_DbLog.p into your
+      <!--
+      <li>To use database logging, copy the file contrib/91_DbLog.pm into your
       modules directory, and change the $dbconn parameter in the file.</li>
+      -->
 
       <li>Each undefined device (FS20, HMS, FHT) will be reported with the
       device name "UNDEFINED". The % parameter will contain the type (FS20,