From caad58856fe74c41bf24db8e9d1e43c30ad4c6ac Mon Sep 17 00:00:00 2001 From: rapster Date: Thu, 1 Oct 2015 15:45:04 +0000 Subject: [PATCH] 93_DbLog: added new setter 'reduceLog' to clean up database. git-svn-id: svn://svn.code.sf.net/p/fhem/code/trunk@9338 2b470e98-0d58-463d-a4d8-8e2adae1ed80 --- fhem/CHANGED | 1 + fhem/FHEM/93_DbLog.pm | 241 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 240 insertions(+), 2 deletions(-) diff --git a/fhem/CHANGED b/fhem/CHANGED index 75ce58c39..1575ac437 100644 --- a/fhem/CHANGED +++ b/fhem/CHANGED @@ -1,5 +1,6 @@ # Add changes at the top of the list. Keep it in ASCII, and 80-char wide. # Do not insert empty lines here, update check depends on it. + - feature: 93_DbLog: added new setter 'reduceLog' to clean up database. - feature: new Modules 36_EleroStick.pm and 36_EleroDrive.pm for Elero shutters - change: 55_InfoPanel.pm: support ReadingsVal in ticker #35228 - bugfix: 31_MilightDevice.pm: improved previousState, fixed log errors diff --git a/fhem/FHEM/93_DbLog.pm b/fhem/FHEM/93_DbLog.pm index b7785a23e..6c672f2ca 100644 --- a/fhem/FHEM/93_DbLog.pm +++ b/fhem/FHEM/93_DbLog.pm @@ -8,6 +8,8 @@ # modified and maintained by Tobias Faust since 2012-06-26 # e-mail: tobias dot faust at online dot de # +# reduceLog() created by Claudiu Schuster (rapster) +# ############################################## package main; @@ -1277,16 +1279,233 @@ DbLog_Get($@) } } + +### DBLog - Historische Werte ausdünnen > Forum #41089 +sub DbLog_reduceLog($@) { + my ($hash,@a) = @_; + my ($ret,$cmd,$row,$filter,$exclude,$c,$day,$hour,$lastHour,$updDate,$updHour,$average,$processingDay,$lastUpdH,%hourlyKnown,%averageHash,@excludeRegex,@dayRows,@averageUpd,@averageUpdD); + my ($dbh,$name,$startTime,$currentHour,$currentDay,$deletedCount,$updateCount,$sum,$rowCount,$excludeCount) = ($hash->{DBH},$hash->{NAME},time(),99,0,0,0,0,0,0); + + if ($a[-1] =~ /^EXCLUDE=(.+:.+)+/i) { + ($filter) = $a[-1] =~ /^EXCLUDE=(.+)/i; + @excludeRegex = split(',',$filter); + } + if (defined($a[3])) { + $average = ($a[3] =~ /average=day/i) ? "AVERAGE=DAY" : ($a[3] =~ /average/i) ? "AVERAGE=HOUR" : 0; + } + Log3($name, 3, "DbLog $name: reduceLog requested with DAYS=$a[2]".(($average || $filter) ? ', ' : '').(($average) ? "$average" : '').(($average && $filter) ? ", " : '').(($filter) ? "EXCLUDE=$filter" : '')); + + if (InternalVal($name,'DBMODEL','') eq 'SQLITE') { $cmd = "datetime('now', '-$a[2] days')"; } + elsif (InternalVal($name,'DBMODEL','') eq 'MYSQL') { $cmd = "DATE_SUB(CURDATE(),INTERVAL $a[2] DAY)"; } + elsif (InternalVal($name,'DBMODEL','') eq 'POSTGRESQL') { $cmd = "NOW() - INTERVAL '$a[2] DAY"; } + else { $ret = 'Unknown database type.'; } + + if ($cmd) { + my $sth_del = $dbh->prepare_cached("DELETE FROM history WHERE (DEVICE=?) AND (READING=?) AND (TIMESTAMP=?) AND (VALUE=?)"); + my $sth_upd = $dbh->prepare_cached("UPDATE history SET TIMESTAMP=?, EVENT=?, VALUE=? WHERE (DEVICE=?) AND (READING=?) AND (TIMESTAMP=?) AND (VALUE=?)"); + my $sth_delD = $dbh->prepare_cached("DELETE FROM history WHERE (DEVICE=?) AND (READING=?) AND (TIMESTAMP=?)"); + my $sth_updD = $dbh->prepare_cached("UPDATE history SET TIMESTAMP=?, EVENT=?, VALUE=? WHERE (DEVICE=?) AND (READING=?) AND (TIMESTAMP=?)"); + my $sth_get = $dbh->prepare("SELECT TIMESTAMP,DEVICE,'',READING,VALUE FROM history WHERE TIMESTAMP < $cmd ORDER BY TIMESTAMP ASC"); # '' was EVENT, no longer in use + $sth_get->execute(); + + do { + $row = $sth_get->fetchrow_arrayref || ['0000-00-00 00:00:00','D','','R','V']; # || execute last-day dummy + $ret = 1; + ($day,$hour) = $row->[0] =~ /-(\d{2})\s(\d{2}):/; + $rowCount++ if($day != 00); + if ($day != $currentDay) { + if ($currentDay) { # false on first executed day + if (scalar @dayRows) { + ($lastHour) = $dayRows[-1]->[0] =~ /(.*\d+\s\d{2}):/; + $c = 0; + for my $delRow (@dayRows) { + $c++ if($day != 00 || $delRow->[0] !~ /$lastHour/); + } + if($c) { + $deletedCount += $c; + Log3($name, 3, "DbLog $name: reduceLog deleting $c records of day: $processingDay"); + $dbh->{RaiseError} = 1; + $dbh->{PrintError} = 0; + $dbh->begin_work(); + eval { + for my $delRow (@dayRows) { + if($day != 00 || $delRow->[0] !~ /$lastHour/) { + Log3($name, 5, "DbLog $name: DELETE FROM history WHERE (DEVICE=$delRow->[1]) AND (READING=$delRow->[3]) AND (TIMESTAMP=$delRow->[0]) AND (VALUE=$delRow->[4])"); + $sth_del->execute(($delRow->[1], $delRow->[3], $delRow->[0], $delRow->[4])); + } + } + }; + if ($@) { + Log3($hash->{NAME}, 3, "DbLog $name: reduceLog ! FAILED ! for day $processingDay"); + $dbh->rollback(); + $ret = 0; + } else { + $dbh->commit(); + } + $dbh->{RaiseError} = 0; + $dbh->{PrintError} = 1; + } + @dayRows = (); + } + + if ($ret && defined($a[3]) && $a[3] =~ /average/i) { + $dbh->{RaiseError} = 1; + $dbh->{PrintError} = 0; + $dbh->begin_work(); + eval { + push(@averageUpd, {%hourlyKnown}) if($day != 00); + + $c = 0; + for my $hourHash (@averageUpd) { # Only count for logging... + for my $hourKey (keys %$hourHash) { + $c++ if ($hourHash->{$hourKey}->[0] && scalar(@{$hourHash->{$hourKey}->[4]}) > 1); + } + } + $updateCount += $c; + Log3($name, 3, "DbLog $name: reduceLog (hourly-average) updating $c records of day: $processingDay") if($c); # else only push to @averageUpdD + + for my $hourHash (@averageUpd) { + for my $hourKey (keys %$hourHash) { + if ($hourHash->{$hourKey}->[0]) { # true if reading is a number + ($updDate,$updHour) = $hourHash->{$hourKey}->[0] =~ /(.*\d+)\s(\d{2}):/; + if (scalar(@{$hourHash->{$hourKey}->[4]}) > 1) { # true if reading has multiple records this hour + for (@{$hourHash->{$hourKey}->[4]}) { $sum += $_; } + $average = sprintf('%.3f', $sum/scalar(@{$hourHash->{$hourKey}->[4]}) ); + $sum = 0; + Log3($name, 5, "DbLog $name: UPDATE history SET TIMESTAMP=$updDate $updHour:30:00, EVENT='rl_av_h', VALUE=$average WHERE DEVICE=$hourHash->{$hourKey}->[1] AND READING=$hourHash->{$hourKey}->[3] AND TIMESTAMP=$hourHash->{$hourKey}->[0] AND VALUE=$hourHash->{$hourKey}->[4]->[0]"); + $sth_upd->execute(("$updDate $updHour:30:00", 'rl_av_h', $average, $hourHash->{$hourKey}->[1], $hourHash->{$hourKey}->[3], $hourHash->{$hourKey}->[0], $hourHash->{$hourKey}->[4]->[0])); + push(@averageUpdD, ["$updDate $updHour:30:00", 'rl_av_h', $average, $hourHash->{$hourKey}->[1], $hourHash->{$hourKey}->[3], $updDate]) if (defined($a[3]) && $a[3] =~ /average=day/i); + } else { + push(@averageUpdD, [$hourHash->{$hourKey}->[0], $hourHash->{$hourKey}->[2], $hourHash->{$hourKey}->[4]->[0], $hourHash->{$hourKey}->[1], $hourHash->{$hourKey}->[3], $updDate]) if (defined($a[3]) && $a[3] =~ /average=day/i); + } + } + } + } + }; + if ($@) { + Log3($hash->{NAME}, 3, "DbLog $name: reduceLog average=hour ! FAILED ! for day $processingDay"); + $dbh->rollback(); + @averageUpdD = (); + } else { + $dbh->commit(); + } + $dbh->{RaiseError} = 0; + $dbh->{PrintError} = 1; + @averageUpd = (); + } + + if (defined($a[3]) && $a[3] =~ /average=day/i && scalar(@averageUpdD) && $day != 00) { + $dbh->{RaiseError} = 1; + $dbh->{PrintError} = 0; + $dbh->begin_work(); + eval { + for (@averageUpdD) { + push(@{$averageHash{$_->[3].$_->[4]}->{tedr}}, [$_->[0], $_->[1], $_->[3], $_->[4]]); + $averageHash{$_->[3].$_->[4]}->{sum} += $_->[2]; + $averageHash{$_->[3].$_->[4]}->{date} = $_->[5]; + } + + $c = 0; + for (keys %averageHash) { + if(scalar @{$averageHash{$_}->{tedr}} == 1) { + delete $averageHash{$_}; + } else { + $c += (scalar(@{$averageHash{$_}->{tedr}}) - 1); + } + } + $deletedCount += $c; + $updateCount += keys(%averageHash); + + Log3($name, 3, "DbLog $name: reduceLog (daily-average) updating ".(keys %averageHash).", deleting $c records of day: $processingDay") if(keys %averageHash); + for my $reading (keys %averageHash) { + $average = sprintf('%.3f', $averageHash{$reading}->{sum}/scalar(@{$averageHash{$reading}->{tedr}})); + $lastUpdH = pop @{$averageHash{$reading}->{tedr}}; + for (@{$averageHash{$reading}->{tedr}}) { + Log3($name, 5, "DbLog $name: DELETE FROM history WHERE DEVICE='$_->[2]' AND READING='$_->[3]' AND TIMESTAMP='$_->[0]'"); + $sth_delD->execute(($_->[2], $_->[3], $_->[0])); + } + Log3($name, 5, "DbLog $name: UPDATE history SET TIMESTAMP=$averageHash{$reading}->{date} 12:00:00, EVENT='rl_av_d', VALUE=$average WHERE (DEVICE=$lastUpdH->[2]) AND (READING=$lastUpdH->[3]) AND (TIMESTAMP=$lastUpdH->[0])"); + $sth_updD->execute(($averageHash{$reading}->{date}." 12:00:00", 'rl_av_d', $average, $lastUpdH->[2], $lastUpdH->[3], $lastUpdH->[0])); + } + }; + if ($@) { + Log3($hash->{NAME}, 3, "DbLog $name: reduceLog average=day ! FAILED ! for day $processingDay"); + $dbh->rollback(); + } else { + $dbh->commit(); + } + $dbh->{RaiseError} = 0; + $dbh->{PrintError} = 1; + } + %averageHash = (); + %hourlyKnown = (); + @averageUpd = (); + @averageUpdD = (); + $currentHour = 99; + } + $currentDay = $day; + } + + if ($hour != $currentHour) { # forget records from last hour, but remember these for average + if (defined($a[3]) && $a[3] =~ /average/i && keys(%hourlyKnown)) { + push(@averageUpd, {%hourlyKnown}); + } + %hourlyKnown = (); + $currentHour = $hour; + } + if (defined $hourlyKnown{$row->[1].$row->[3]}) { # remember first readings for device per h, other can be deleted + push(@dayRows, [@$row]); + if (defined($a[3]) && $a[3] =~ /average/i && defined($row->[4]) && $row->[4] =~ /^-?(?:\d+(?:\.\d*)?|\.\d+)$/ && $hourlyKnown{$row->[1].$row->[3]}->[0]) { + if ($hourlyKnown{$row->[1].$row->[3]}->[0]) { + push(@{$hourlyKnown{$row->[1].$row->[3]}->[4]}, $row->[4]); + } + } + } else { + $exclude = 0; + for (@excludeRegex) { + $exclude = 1 if("$row->[1]:$row->[3]" =~ /^$_$/); + } + if ($exclude) { + $excludeCount++ if($day != 00); + } else { + $hourlyKnown{$row->[1].$row->[3]} = (defined($row->[4]) && $row->[4] =~ /^-?(?:\d+(?:\.\d*)?|\.\d+)$/) ? [$row->[0],$row->[1],$row->[2],$row->[3],[$row->[4]]] : [0]; + } + } + $processingDay = (split(' ',$row->[0]))[0]; + } while( $day != 00 ); + + my $result = "Rows processed: $rowCount, deleted: $deletedCount" + .((defined($a[3]) && $a[3] =~ /average/i)? ", updated: $updateCount" : '') + .(($excludeCount)? ", excluded: $excludeCount" : '') + .", time: ".sprintf('%.2f',time() - $startTime)."sec"; + Log3($name, 3, "DbLog $name: reduceLog executed. $result"); + readingsSingleUpdate($hash, 'lastReduceLogResult', $result ,1); + $ret = "reduceLog executed. $result"; + } + + return $ret; +} + + sub DbLog_Set($@) { - my ($hash, @a) = @_; + my ($hash, @a) = @_; my $name = $hash->{NAME}; - my $usage = "Unknown argument, choose one of reopen:noArg rereadcfg:noArg count:noArg deleteOldDays userCommand"; + my $usage = "Unknown argument, choose one of reduceLog reopen:noArg rereadcfg:noArg count:noArg deleteOldDays userCommand"; return $usage if(int(@a) < 2); my $dbh = $hash->{DBH}; my $ret; given ($a[1]) { + when ('reduceLog') { + if (defined $a[2] && $a[2] =~ /^\d+$/) { + $ret = DbLog_reduceLog($hash,@a); + } else { + Log3($name, 1, "DbLog $name: reduceLog error, no given."); + $ret = "reduceLog error, no given."; + } + } + when ('reopen') { Log3($name, 4, "DbLog $name: Reopen requested."); $dbh->commit() if(! $dbh->{AutoCommit}); @@ -1822,6 +2041,15 @@ sub dbReadings($@) { set <name> deleteOldDays <n>

    Delete records from history older than <n> days. Number of deleted record will be written into reading lastRowsDeleted.

+ set <name> reduceLog <n> [average[=day]] [exclude=deviceRegExp1:ReadingRegExp1,deviceRegExp2:ReadingRegExp2,...]

+
    Reduce records older than <n> days to one record each hour (the 1st) per device & reading.
    + CAUTION: It is strongly recommended to check if the default INDEX 'Search_Idx' exists on the table 'history'!
    + The execution of this command may take (without INDEX) extremely long, FHEM will be completely blocked after issuing the command to completion!
    + With the optional argument 'average' not only the records will be reduced, but all numerical values of an hour will be reduced to a single average.
    + With the optional argument 'average=day' not only the records will be reduced, but all numerical values of a day will be reduced to a single average. (implies 'average')
    + You can optional set the last argument to "EXCLUDE=deviceRegExp1:ReadingRegExp1,deviceRegExp2:ReadingRegExp2,...." to exclude device/readings from reduceLog
    +

+ set <name> userCommand <validSqlStatement>

    DO NOT USE THIS COMMAND UNLESS YOU REALLY (REALLY!) KNOW WHAT YOU ARE DOING!!!

    Perform any (!!!) sql statement on connected database. Useercommand and result will be written into corresponding readings.
    @@ -2172,6 +2400,15 @@ sub dbReadings($@) { set <name> deleteOldDays <n>

      Löscht Datensätze, die älter sind als <n> Tage. Die Anzahl der gelöschten Datensätze wird in das Reading lastRowsDeleted geschrieben.

    + set <name> reduceLog <n> [average[=day]] [exclude=deviceRegExp1:ReadingRegExp1,deviceRegExp2:ReadingRegExp2,...]

    +
      Reduziert historische Datensaetze, die aelter sind als <n> Tage auf einen Eintrag pro Stunde (den ersten) je device & reading.
      + ACHTUNG: Es wird dringend empfohlen zu überprüfen ob der standard INDEX 'Search_Idx' in der Tabelle 'history' existiert!
      + Die Abarbeitung dieses Befehls dauert unter umständen (ohne INDEX) extrem lange, FHEM wird nach absetzen des Befehls bis zur Fertigstellung komplett blockiert!
      + Durch die optionale Angabe von 'average' wird nicht nur die Datenbank bereinigt, sondern alle numerischen Werte einer Stunde werden auf einen einzigen Mittelwert reduziert.
      + Durch die optionale Angabe von 'average=day' wird nicht nur die Datenbank bereinigt, sondern alle numerischen Werte eines Tages auf einen einzigen Mittelwert reduziert. (impliziert 'average')
      + Optional kann als letzer Parameter "EXCLUDE=deviceRegExp1:ReadingRegExp1,deviceRegExp2:ReadingRegExp2,...." angegeben werden um device/reading Kombinationen von reduceLog auszuschließen.
      +

    + set <name> userCommand <validSqlStatement>

      BENUTZE DIESE FUNKTION NUR, WENN DU WIRKLICH (WIRKLICH!) WEISST, WAS DU TUST!!!

      Führt einen beliebigen (!!!) sql Befehl in der Datenbank aus. Der Befehl und ein zurückgeliefertes Ergebnis werden in entsprechende Readings geschrieben.