From fab31ae7949e8fec6cdc66730e39dc057949daf7 Mon Sep 17 00:00:00 2001 From: neubert Date: Sat, 17 Dec 2016 15:17:38 +0000 Subject: [PATCH] 66_ECMD, 67_ECMDDevice: - if split is used, the strings at which the messages are split are still part of the messages - no default attributes for requestSeparator and responseSeparator - input of raw data as perl-encoded string (for setting attributes) - be more verbose and explicit at loglevel 5 - documentation corrected and amended git-svn-id: https://svn.fhem.de/fhem/trunk@12796 2b470e98-0d58-463d-a4d8-8e2adae1ed80 --- fhem/CHANGED | 8 ++ fhem/FHEM/66_ECMD.pm | 231 +++++++++++++++++++++++++++---------- fhem/FHEM/67_ECMDDevice.pm | 49 +++++--- 3 files changed, 210 insertions(+), 78 deletions(-) diff --git a/fhem/CHANGED b/fhem/CHANGED index fb582feb6..560680f04 100644 --- a/fhem/CHANGED +++ b/fhem/CHANGED @@ -1,5 +1,13 @@ # Add changes at the top of the list. Keep it in ASCII, and 80-char wide. # Do not insert empty lines here, update check depends on it. + - feature: 66_ECMD, 67_ECMDDevice: + - if split is used, the strings at which the messages are split + are still part of the messages + - no default attributes for requestSeparator and responseSeparator + - input of raw data as perl-encoded string (for setting + attributes) + - be more verbose and explicit at loglevel 5 + - documentation corrected and amended - feature: 57_Calendar: BYDAY: recognizes and honors one or several weekdays with and without prefix (e.g. -1SU, 2MO) with MONTHLY - bugfix: contrib/Widgets/DateTimePicker/fhemweb_datetime.js: parameter diff --git a/fhem/FHEM/66_ECMD.pm b/fhem/FHEM/66_ECMD.pm index f960e6745..4e2d9f3d3 100644 --- a/fhem/FHEM/66_ECMD.pm +++ b/fhem/FHEM/66_ECMD.pm @@ -25,9 +25,30 @@ package main; -# -# Potential future extensions: add support for PARTIALly received datagrams -# http://forum.fhem.de/index.php/topic,24280.msg174330.html#msg174330 + +=for comment + +General rule: + +ECMD handles raw data, i.e. data that might contain control and non-printable characters. +User input for raw data, e.g. setting attributes, and display of raw data is perl-encoded. +Perl-encoded raw data in logs is not enclosed in double quotes. + +A carriage return/line feed (characters 13 and 10) is encoded as +\r\n +and logged as +\r\n (\010\012) + +Decoding is handled by dq(). Encoding is handled by cq(). + +changes as of 27 Nov 2016: +- if split is used, the strings at which the messages are split are still part of the messages +- no default attributes for requestSeparator and responseSeparator +- input of raw data as perl-encoded string (for setting attributes) +- be more verbose and explicit at loglevel 5 +- documentation corrected and amended + +=cut use strict; @@ -84,7 +105,9 @@ ECMD_Define($$) return $msg; } - $attr{$name}{"requestSeparator"}= "\000"; + $hash->{fhem}{".requestSeparator"}= undef; + $hash->{fhem}{".responseSeparator"}= undef; + $hash->{fhem}{".split"}= undef; DevIo_CloseDev($hash); @@ -136,33 +159,37 @@ ECMD_DoInit($) ##################################### sub -dq($) +oq($) { -=for comment - '\a' => "\\a", - '\e' => "\\e", - '\f' => "\\f", - '\n' => "\\n", - '\r' => "\\r", - '\t' => "\\t", - ); - - $s =~ s/\\/\\\\/g; - foreach my $regex (keys %escSequences) { - $s =~ s/$regex/$escSequences{$regex}/g; - } - $s =~ s/([\000-\037])/sprintf("\\%03o", ord($1))/eg; - - - -=cut - - my ($s)= @_; - $s= "" unless(defined($s)); - return "\"" . escapeLogLine($s) . "\""; + return join("", map { sprintf("\\%03o", ord($_)) } split("", $s)); } +sub +dq($) +{ + my ($s)= @_; + return defined($s) ? escapeLogLine($s) . " (" . oq($s) . ")" : ""; +} + +sub +cq($) +{ + my ($s)= @_; + + $s =~ s/\\(\d)(\d)(\d)/chr($1*64+$2*8+$3)/eg; + $s =~ s/\\a/\a/g; + $s =~ s/\\e/\e/g; + $s =~ s/\\f/\f/g; + $s =~ s/\\n/\n/g; + $s =~ s/\\r/\r/g; + $s =~ s/\\t/\t/g; + $s =~ s/\\\\/\\/g; + + return $s; +} + +##################################### sub ECMD_Log($$$) { @@ -170,7 +197,7 @@ ECMD_Log($$$) my $name= $hash->{NAME}; $loglevel= AttrVal($name, "logTraffic", undef) unless(defined($loglevel)); return unless(defined($loglevel)); - Log3 $hash, $loglevel , "$name: $logmsg"; + Log3 $hash, $loglevel, "$name: $logmsg"; } ##################################### @@ -498,8 +525,10 @@ ECMD_Attr($@) my @a = @_; my $hash= $defs{$a[1]}; + my $name= $hash->{NAME}; - if($a[0] eq "set" && $a[2] eq "classdefs") { + if($a[0] eq "set") { + if($a[2] eq "classdefs") { my @classdefs= split(/:/,$a[3]); delete $hash->{fhem}{classDefs}; @@ -507,6 +536,30 @@ ECMD_Attr($@) my ($classname,$filename)= split(/=/,$classdef,2); ECMD_EvalClassDef($hash, $classname, $filename); } + } elsif($a[2] eq "requestSeparator") { + my $c= cq($a[3]); + $hash->{fhem}{".requestSeparator"}= $c; + Log3 $hash, 5, "$name: requestSeparator set to " . dq($c); + } elsif($a[2] eq "responseSeparator") { + my $c= cq($a[3]); + $hash->{fhem}{".responseSeparator"}= $c; + Log3 $hash, 5, "$name: responseSeparator set to " . dq($c); + } elsif($a[2] eq "split") { + my $c= cq($a[3]); + $hash->{fhem}{".split"}= $c; + Log3 $hash, 5, "$name: split set to " . dq($c); + } + } elsif($a[0] eq "del") { + if($a[2] eq "requestSeparator") { + $hash->{fhem}{".requestSeparator"}= undef; + Log3 $hash, 5, "$name: requestSeparator deleted"; + } elsif($a[2] eq "responseSeparator") { + $hash->{fhem}{".responseSeparator"}= undef; + Log3 $hash, 5, "$name: responseSeparator deleted"; + } elsif($a[2] eq "split") { + $hash->{fhem}{".split"}= undef; + Log3 $hash, 5, "$name: split deleted"; + } } return undef; @@ -572,19 +625,19 @@ ECMD_Write($$$) my $name= $hash->{NAME}; my $answer; my $ret= ""; - my $requestSeparator= AttrVal($name, "requestSeparator", undef); - my $responseSeparator= AttrVal($name, "responseSeparator", ""); + my $requestSeparator= $hash->{fhem}{".requestSeparator"}; + my $responseSeparator= $hash->{fhem}{".responseSeparator"}; my @ecmds; if(defined($requestSeparator)) { @ecmds= split $requestSeparator, $msg; } else { push @ecmds, $msg; } - ECMD_Log $hash, 5, "command split into " . ($#ecmds+1) . " parts." if($#ecmds>0); + ECMD_Log $hash, 5, "command split into " . ($#ecmds+1) . " parts, requestSeparator is " . + dq($requestSeparator) if($#ecmds>0); foreach my $ecmd (@ecmds) { ECMD_Log $hash, 5, "sending command " . dq($ecmd); my $msg .= $ecmd; - #$msg.= "\n" unless($nonl); if(defined($expect)) { $answer= ECMD_SimpleExpect($hash, $msg, $expect); $answer= "" unless(defined($answer)); @@ -603,18 +656,21 @@ ECMD_Write($$$) 1; =pod +=item device +=item summary configurable request/response-like communication (physical device) +=item summary_DE konfigurierbare Frage/Antwort-Kommunikation (physisches Gerät) =begin html

ECMD

    Any physical device with request/response-like communication capabilities - over a TCP connection can be defined as ECMD device. A practical example + over a serial line or TCP connection can be defined as ECMD device. A practical example of such a device is the AVR microcontroller board AVR-NET-IO from Pollin with ECMD-enabled Ethersex firmware. The original - NetServer firmware from Pollin works as well.

    + NetServer firmware from Pollin works as well. There is a plenitude of use cases.

    A physical ECMD device can host any number of logical ECMD devices. Logical devices are defined as ECMDDevices in fhem. @@ -635,11 +691,31 @@ ECMD_Write($$$)

    Note: this module requires the Device::SerialPort or Win32::SerialPort module - if the module is connected via serial Port or USB. + if the module is connected via serial Port or USB.

    + + + Character coding

    + + ECMD is suited to process any character including non-printable and control characters. + User input for raw data, e.g. for setting attributes, and the display of raw data, e.g. in the log, + is perl-encoded according to the following table (ooo stands for a three-digit octal number):
    + + + + + + + + + + + +
    characteroctalcode
    Bell007\a
    Backspace008\008
    Escape033\e
    Formfeed014\f
    Newline012\n
    Return015\r
    Tab011\t
    backslash134\134 or \\
    anyooo\ooo

    + In user input, use \134 for backslash to avoid conflicts with the way FHEM handles continuation lines.

    - Define + Define

      define <name> ECMD telnet <IPAddress:Port>

      or

      @@ -653,13 +729,13 @@ ECMD_Write($$$)
        define AVRNETIO ECMD telnet 192.168.0.91:2701
        define AVRNETIO ECMD serial /dev/ttyS0
        - define AVRNETIO ECMD serial /sev/ttyUSB0@38400
        + define AVRNETIO ECMD serial /dev/ttyUSB0@38400

    - Set + Set

      set <name> classdef <classname> <filename>

      @@ -670,7 +746,7 @@ ECMD_Write($$$)

      Example:
        - define AVRNETIO classdef /etc/fhem/ADC.classdef
        + set AVRNETIO classdef /etc/fhem/ADC.classdef

      set <name> reopen @@ -682,7 +758,7 @@ ECMD_Write($$$) - Get + Get

        get <name> raw <command>

        @@ -699,45 +775,76 @@ ECMD_Write($$$)

        - Attributes -

        + Attributes

        • classdefs
          A colon-separated list of <classname>=<filename>. The list is automatically updated if a class definition is added. You can - directly set the attribute.
          • -
        • split
          + directly set the attribute. Example: attr myECMD classdefs ADC=/etc/fhem/ADC.classdef:GPIO=/etc/fhem/AllInOne.classdef
          • +
        • split <separator>
          Some devices send several readings in one transmission. The split attribute defines the separator to split such transmissions into separate messages. The regular expression for matching a reading is then applied to each message in turn. After splitting, the separator - is not part of the single messages. - Example: attr myECMD \n splits foo 12\nbar off into - foo 12 and bar off.
          • -
        • logTraffic <loglevel>
          Enables logging of sent and received datagrams with the given loglevel. Control characters in the logged datagrams are escaped, i.e. a double backslash is shown for a single backslash, \n is shown for a line feed character, etc.
          • -
        • timeout <seconds>
          Time in seconds to wait for a reply from the physical ECMD device before FHEM assumes that something has gone wrong. The default is 3 seconds if this attribute is not set.
          • -
        • partial <seconds>
          Some physical ECMD devices split readings and replies into several transmissions. If the partial attribute is set, this behavior is accounted for as follows: (a) If a reply is expected for a get or set command, FHEM collects transmissions from the physical ECMD device until either the reply matches the expected reply or the time in seconds given with the partial attribute has expired. (b) If a spontaneous transmission does not match the regular expression for any reading, the transmission is recorded and prepended to the next transmission. If the line is quiet for longer than the time in seconds given with the partial attribute, the recorded transmission is discarded. Use regular expressions that produce exact matches.
          • -
        • requestSeparator
          - A single request from FHEM to the device might need to be split in several datagrams. A command string is split at all - occurrences of the requestSeparator. The requestSeparator itself is removed from the command string and thus - not part of the request. - This attribute is set by default. It defaults to the value \000 (octal representation of control char with code zero). - To disable this feature, delete the attribute from the device's attribute list. + is still part of the single messages. Separator can be a single- or multi-character string, + e.g. \n or \r\n. + Example: attr myECMD split \n splits foo 12\nbar off\n into + foo 12\n and bar off\n.
          • +
        • logTraffic <loglevel>
          Enables logging of sent and received datagrams with the given loglevel. Control characters in the logged datagrams are escaped, i.e. a double backslash is shown for a single backslash, \n is shown for a line feed character, etc.
          • +
        • timeout <seconds>
          Time in seconds to wait for a response from the physical ECMD device before FHEM assumes that something has gone wrong. The default is 3 seconds if this attribute is not set.
          • +
        • partial <seconds>
          Some physical ECMD devices split responses into several transmissions. If the partial attribute is set, this behavior is accounted for as follows: (a) If a response is expected for a get or set command, FHEM collects transmissions from the physical ECMD device until either the response matches the expected response (reading ... match ... in the class definition) or the time in seconds given with the partial attribute has expired. (b) If a spontaneous transmission does not match the regular expression for any reading, the transmission is recorded and prepended to the next transmission. If the line is quiet for longer than the time in seconds given with the partial attribute, the recorded transmission is discarded. Use regular expressions that produce exact matches of the complete response (after combining partials and splitting).
          • +
        • requestSeparator <separator>
          + A single command from FHEM to the device might need to be broken down into several requests. + A command string is split at all + occurrences of the request separator. The request separator itself is removed from the command string and thus is + not part of the request. The default is to have no response separator. Use a request separator that does not occur in the actual request.
          • -
        • responseSeparator
          - In order to identify the single responses from the device to FHEM for each part of a split command, a responseSeparator - can be appended to the response to each part. The responseSeparator is only appended to commands split by means of a - requestSeparator. The default is to have no responseSeparator, i.e. responses are simply concatenated. +
        • responseSeparator <separator>
          + In order to identify the single responses from the device for each part of the command broken down by request separators, a response separator can be appended to the response to each single request. + The response separator is only appended to commands split by means of a + request separator. The default is to have no response separator, i.e. responses are simply concatenated. Use a response separator that does not occur in the actual response.
        • verbose


        + Separators +

        + When to use the split and partial attributes?

        + + Set the partial attribute in combination with reading ... match ... in the class definition, if you receive datagrams with responses which are broken into several transmissions, like resp followed by onse\r\n.

        + + Set the split attribute if you + receive several responses in one transmission, like reply1\r\nreply2\r\n.

        + + When to use the requestSeparator and responseSeparator attributes?

        + + Set the requestSeparator attribute, if you want to send several requests in one command, with one transmission per request. The strings sent to the device for set and get commands + as defined in the class definition are broken down into several request/response + interactions with the physical device. The request separator is not sent to the physical device.

        + + Set the responseSeparator attribute to separate the responses received for a command + broken down into several requests by means of a request separator. This is useful for easier postprocessing.

        + + Example: you want to send the requests request1 and request2 in one command. The + physical device would respond with response1 and response2 respectively for each + of the requests. You set the request separator to \000 and the response separator to \001 and you define + the command as request1\000request2\000. The command is broken down into request1 + and request2. request1 is sent to the physical device and response1 + is received, followed by sending request2 and receiving response2. The final + result is response1\001response2\001.

        + + You can think of this feature as of a macro. Splitting and partial matching is still done per single + request/response within the macro.

        + Datagram monitoring and matching

        Data to and from the physical device is processed as is. In particular, if you need to send a line feed you have to explicitely send a \n control character. On the other hand, control characters like line feeds are not stripped from the data received. This needs to be considered when defining a class definition.

        - For debugging purposes, especially when designing a class definition, it is advisable to turn traffic logging on. Use attr myECMD logTraffic 3 to log all data to and from the physical device at level 3. A typical response might look like 21.2\n, i.e. a floating point number followed by a newline.

        + For debugging purposes, especially when designing a class definition, it is advisable to turn traffic logging on. Use attr myECMD logTraffic 3 to log all data to and from the physical device at level 3.

        + + Datagrams and attribute values are logged with non-printable and control characters encoded as here followed by the octal representation in parantheses. + Example: #!foo\r\n (\043\041\146\157\157\015\012).

        Data received from the physical device is processed as it comes in chunks. If for some reason a datagram from the device is split in transit, pattern matching and processing will most likely fail. You can use the partial attribute to make FHEM collect and recombine the chunks.

        diff --git a/fhem/FHEM/67_ECMDDevice.pm b/fhem/FHEM/67_ECMDDevice.pm index c95a2bb8f..4aa7f745f 100644 --- a/fhem/FHEM/67_ECMDDevice.pm +++ b/fhem/FHEM/67_ECMDDevice.pm @@ -337,21 +337,27 @@ ECMDDevice_Parse($$) $IOhash->{fhem}{partial}{msg}= ""; } + my $partial= $IOhash->{fhem}{partial}{msg}; + #Debug "$name: partial message \"" . escapeLogLine($IOhash->{fhem}{partial}{msg}) . "\" recorded at $ts"; - if($IOhash->{fhem}{partial}{msg} ne "") { + if($partial ne "") { # clear partial message if expired my $timeout= AttrVal($name, "partial", 1); my $t0= $IOhash->{fhem}{partial}{ts}; if($ts-$t0> $timeout) { - $IOhash->{fhem}{partial}{msg}= ""; - #Debug "$name: partial message expired."; + Log3 $IOhash, 5, "$name: partial message " . dq($partial) . " expired."; + $partial= ""; + $IOhash->{fhem}{partial}{msg}= $partial; } } # prepend to recently received message $IOhash->{fhem}{partial}{ts}= $ts; - $message= $IOhash->{fhem}{partial}{msg} . $message; - $IOhash->{fhem}{partial}{msg}= ""; + if($partial ne "") { + Log3 $IOhash, 5, "$name: merging partial message " . dq($partial) . " and " . dq($message); + $message= $partial . $message; + $IOhash->{fhem}{partial}{msg}= ""; + } } else { @@ -364,8 +370,15 @@ ECMDDevice_Parse($$) #Debug "$name: analyzing \"" . escapeLogLine($message) . "\"."; my @msgs; - if(defined(AttrVal($name, "split", undef))) { - @msgs= split(AttrVal($name, "split", undef), $message); + my $splitter= $IOhash->{fhem}{".split"}; + if(defined($splitter)) { + #Debug "Splitting " . dq($message) . " at " . dq($splitter); + @msgs= split(/(?<=$splitter)/, $message); # http://stackoverflow.com/questions/14907772/split-but-keep-delimiter + + #Debug scalar(@msgs) . " part(s)"; + Log3 $IOhash, 5, "$name: " . dq($message) . " split into " . scalar(@msgs) . " parts" + if(scalar(@msgs)>1); + #Debug "Split done."; } else { push @msgs, $message; } @@ -376,6 +389,7 @@ ECMDDevice_Parse($$) foreach my $msg (@msgs) { #Debug "$name: trying to find a match for \"" . escapeLogLine($msg) ."\""; + Log3 $IOhash, 5, "$name: trying to match message " . dq($msg); $msgMatched= 0; # walk over all clients foreach my $d (keys %defs) { @@ -393,7 +407,7 @@ ECMDDevice_Parse($$) #Debug " Trying to match reading $r with regular expression \"$regex\" (device $d, classdef $classname, reading $r)."; if($msg =~ m/^$regex$/) { # we found a match - Log3 $IOhash, 5, "$name: match regex $regex for reading $r of device $d with class $classname"; + Log3 $IOhash, 5, "$name: " . dq($msg) . " matches regex $regex for reading $r of device $d with class $classname"; $msgMatched++; push @matches, $d; my $command= ECMDDevice_GetCachedReadingsCommand($hash, $classDef, $r); @@ -416,6 +430,7 @@ ECMDDevice_Parse($$) $IOhash->{fhem}{partial}{msg}= ""; } $IOhash->{fhem}{partial}{msg}.= $lastMsg; # append unmatched message + Log3 $IOhash, 5, "$name: partial message " . dq($lastMsg) . " kept"; #Debug "$name: partial message \"" . escapeLogLine($IOhash->{fhem}{partial}{msg}) . "\" kept."; } @@ -522,8 +537,8 @@ ECMDDevice_Define($$) =pod =item device -=item summary user-defined device communicating through ECMD -=item summary_DE ein benutzerdefiniertes Gerät, welches über ECMD kommuniziert +=item summary user-defined device communicating through ECMD (logical device) +=item summary_DE benutzerdefiniertes via ECMD kommunizierendes Gerät (logisches Gerät) =begin html @@ -636,7 +651,7 @@ ECMDDevice_Define($$) In the fhem configuration file or on the fhem command line we do the following:

        define AVRNETIO ECMD telnet 192.168.0.91:2701 # define the physical device
        - set AVRNETIO classdef ADC /etc/fhem/ADC.classdef # define the device class ADC
        + attr AVRNETIO classdefs ADC=/etc/fhem/ADC.classdef # define the device class ADC
        define myADC ECDMDevice ADC # define the logical device myADC with device class ADC
        get myADC value 1 # retrieve the value of analog/digital converter number 1
         @@ -667,8 +682,9 @@ ECMDDevice_Define($$)
        In the fhem configuration file or on the fhem command line we do the following:

        - define AVRNETIO ECMD telnet 192.168.0.91:2701 # define the physical device
        - set AVRNETIO classdef relais /etc/fhem/relais.classdef # define the device class relais
        + define AVRNETIO ECMD telnet 192.168.0.91:2701 # define the physical device
        + attr AVRNETIO classdefs relais=/etc/fhem/relais.classdef # define the device class relais
        + attr AVRNETIO requestSeparator \000
        define myRelais ECMDDevice 8 # define the logical device myRelais with pin mask 8
        set myRelais on # execute the "on" command
         @@ -686,12 +702,13 @@ ECMDDevice_Define($$)

      These lines are sent as a plain ethersex commands to the AVR-NET-IO one by one. After - each line the answer from the physical device is read back. They are concatenated with \000 chars and returned - for further processing by the postproc command. + each line the answer from the physical device is read back. They are concatenated and returned + for further processing by the postproc command. For any of the four plain ethersex commands, the AVR-NET-IO returns the string OK\n. They are concatenated. The postprocessor takes the result from $_, substitutes it by the string success if it is OK\nOK\nOK\nOK\n, and then either - returns the string ok or the string error. + returns the string ok or the string error. If the responseSeparator was set to \000, + the result string would be OK\n\000OK\n\000OK\n\000OK\n\000 instead of OK\nOK\nOK\nOK\n.