Send wind even if it hasn't changed
[dweather.git] / loop.pl
diff --git a/loop.pl b/loop.pl
index e4a1fb69eb2c341a26d3af625c1ad03ecb6ca336..e627dde79127b8a86f3f66a2cec460a97e6e46dc 100755 (executable)
--- a/loop.pl
+++ b/loop.pl
@@ -3,15 +3,28 @@ use strict;
 
 use v5.10.1;
 
+use Mojolicious::Lite;
 use Serial;
 use Mojo::IOLoop;
 use Mojo::IOLoop::Stream;
+use Mojo::Transaction::WebSocket;
 #use Mojo::JSON  qw(decode_json encode_json);
 use JSON;
 use Debug;
 use SMGLog;
+use Math::Round qw(nearest);
+use File::Copy;
+use Data::Random qw(rand_chars);
+use IO::File;
 
+use constant pi => 3.14159265358979; 
+
+my $randomfn = '/dev/urandom';
 my $devname = "/dev/davis";
+my $datafn = ".loop_data";
+my $dataf;
+
+my $poll_interval = 2.5;
 my $rain_mult = 0.2;                           # 0.1 or 0.2 mm or 0.01 inches
 my $tid;
 my $rid;
@@ -20,17 +33,20 @@ my $nlcount;
 my $state = "ready";
 my $buf;
 my $dbg;
-my $last_reading;
-my $s;                                                 # the serial port Mojo::IOLoop::Stream
-our $ending = 0;
+our $ser;                                                      # the serial port Mojo::IOLoop::Stream
+our $ob;                                                       # the Serial Port filehandle
+my $last_min_h;
+my $last_hour_h;
 
 our $json = JSON->new->canonical(1);
+our $WS = {};                                  # websocket connections
 
-our $last_min = int(time/60)*60;
-our $last_hour = 0;
-our $last_rain_hour;
-our $last_rain_min;
-our $last_rain;
+our $ld = {};
+our @last10minsr = ();
+our @last5daysh = ();
+our $windmins = 2;                             # no of minutes of wind data for the windrose
+our $histdays = 5;                             # no of days of (half)hour data to search for main graph
+our $updatepermin = 60 / 2.5;  # no of updates per minute
 
 our $loop_count;                               # how many LOOPs we have done, used as start indicator
 
@@ -81,9 +97,52 @@ $bar_trend{60} = "Rising Rapidly";
 
 our $ending = 0;
 
-$SIG{TERM} = $SIG{INT} = sub {++$ending; Mojo::IOLoop->stop;};
+$SIG{TERM} = $SIG{INT} = sub {$ending = 1; Mojo::IOLoop->stop;};
 $SIG{HUP} = 'IGNORE';
 
+
+# WebSocket weather service
+websocket '/weather' => sub {
+  my $c = shift;
+  my $msg = shift;
+  my $tx = $c->tx;
+
+  # Opened
+  app->log->debug('WebSocket opened.');
+  dbg 'WebSocket opened' if isdbg 'chan';
+  $WS->{$tx} = $tx; 
+
+  # send historical data
+  $c->send($ld->{lasthour_h}) if exists $ld->{lasthour_h};
+  $c->send($ld->{lastmin_h}) if exists $ld->{lastmin_h};
+
+  # disable timeout
+  $c->inactivity_timeout(3615);
+  # Incoming message
+  $c->on(
+                message => sub {
+                        my ($c, $msg) = @_;
+                        dbg "websocket: text $msg" if isdbg 'chan';
+                },
+                json => sub {
+                        my ($c, $msg) = @_;
+                        dbg "websocket: json $msg" if isdbg 'chan';
+                }
+  );
+  # Closed
+  $c->on(finish => sub {
+    my ($c, $code, $reason) = @_;
+    app->log->debug("WebSocket closed with status $code.");
+       dbg "websocket closed with status $code" if isdbg 'chan';
+       delete $WS->{$tx};
+  });
+};
+
+get '/' => {template => 'index'};
+
+
 dbginit();
 if (@ARGV) {
        dbgadd(@ARGV);
@@ -94,23 +153,64 @@ dbg '***';
 dbg "*** starting $0";
 dbg '***';
 
-dbg scalar gmtime($last_min);
-dbg scalar gmtime($last_hour);
+read_ld();
+
+my $tnow = time;
+my $dayno = int ($tnow/86400);
+for (my $i = 0-$histdays; $i < 0; ++$i ) {
+       push @last5daysh, grab_history(SMGLog->new("day"), "h", $tnow-(86400*$histdays), $dayno+$i+1); 
+}
+@last10minsr = map {my ($t, $js) = split(/\s/, $_, 2); $js} grab_history(SMGLog->new("debug"), "r", $tnow-(60*$windmins), $dayno);
+dbg sprintf("last5days = %d last10mins = %d", scalar @last5daysh, scalar @last10minsr);
 
-my $dlog = SMGLog->new("day");
+sysopen(R, $randomfn, 0) or die "cannot open $randomfn $!\n";
+my $rs;
+sysread(R, $rs, 8) or die "not enough randomness available\n";
+close R;
+
+app->secrets([qw(Here's something that's really seakrett), $rs]);
+
+our $dlog = SMGLog->new("day");
 $did = Mojo::IOLoop->recurring(1 => sub {$dlog->flushall});
 
-do_reopen($devname);
+dbg "before next tick";
+Mojo::IOLoop->next_tick(sub { loop() });       
+dbg "before app start";
+app->start;
+dbg "after app start";
+
+doclose();
+
+write_ld();
+$dataf->close if $dataf;
+undef $dataf;
+
+
+# move all the files along one 
+cycle_loop_data_files();
 
 dbg '***';
-dbg "*** ending $0";
+dbg "*** ending $0 (\$ending = $ending)";
 dbg '***';
 
 exit 0;
 
+##################################################################################
+       
+sub loop
+{
+       dbg "last_min: " . scalar gmtime($ld->{last_min});
+       dbg "last_hour: " . scalar gmtime($ld->{last_hour});
+       
+       $ser = doopen($devname);
+       start_loop() if $ser;
+}
+
+
+
 sub on_read
 {
-       my ($str, $d) = @_;
+       my ($ser, $d) = @_;
        $buf .= $d;
        $d =~ s/([\%\x00-\x1f\x7f-\xff])/sprintf("%%%02X", ord($1))/eg; 
        dbg "read added '$d' buf lth=" . length $buf if isdbg 'raw';
@@ -120,7 +220,7 @@ sub on_read
                undef $tid;
                undef $buf;
                $nlcount = 0;
-               $s->write("LPS 1 1\n");
+               $ser->write("LPS 1 1\n");
                chgstate("waitloop");
        } elsif ($state eq "waitloop") {
                if ($buf =~ /\x06/) {
@@ -142,17 +242,16 @@ sub start_loop
 {
        $nlcount = 1;
        dbg "start_loop writing $nlcount \\n" if isdbg 'state';
-       $s->write("\n");
+       $ser->write("\n");
        Mojo::IOLoop->remove($tid) if $tid;
        undef $tid;
        $tid = Mojo::IOLoop->recurring(0.6 => sub {
                                                                           if (++$nlcount > 10) {
-                                                                                  dbg "\\n count > 10, closing connection" if isdbg 'chan';
-                                                                                  do_reopen($devname);
+                                                                                  doclose();
                                                                                   return;
                                                                           }
                                                                           dbg "writing $nlcount \\n" if isdbg 'state'; 
-                                                                          $s->write("\n");
+                                                                          $ser->write("\n");
                                                                   });
        chgstate("waitnl");
 }
@@ -163,47 +262,68 @@ sub chgstate
        $state = $_[0];
 }
 
-sub do_reopen
-{
-       my $name = shift;
-       dbg "do reopen on '$name' ending $ending";
-       unless ($ending) {
-               $s = do_open($name);
-               start_loop();
-               chgstate('');
-               $nlcount = 0;
-               Mojo::IOLoop->start unless Mojo::IOLoop->is_running;
-       }
-}
+my $closing;
 
-sub do_open
+sub doopen
 {
        my $name = shift;
 
-       $s->close if $s;
-       undef $s;
-
-       my $ob = Serial->new($name, 19200) || die "$name $!\n";
+    $ob = Serial->new($name, 19200) || die "$name $!\n";
        dbg "streaming $name fileno(" . fileno($ob) . ")" if isdbg 'chan';
        
-       my $str = Mojo::IOLoop::Stream->new($ob);
-       $str->on(error=>sub {dbg "serial $_[1]"; do_reopen($name) unless $ending});
-       $str->on(close=>sub {dbg "serial closing"; do_reopen($name) unless $ending});
-       $str->on(timeout=>sub {dbg "serial timeout";});
-       $str->on(read=>sub {on_read(@_)});
-       $str->start;
+       my $ser = Mojo::IOLoop::Stream->new($ob);
+       $ser->on(error=>sub {dbg "error serial $_[1]"; doclose();});
+       $ser->on(close=>sub {dbg "event close";  doclose();});
+       $ser->on(timeout=>sub {dbg "event serial timeout"; doclose();});
+       $ser->on(read=>sub {on_read(@_)});
+       $ser->start;
+
+    $rid = Mojo::IOLoop->recurring($poll_interval => sub {
+                                       start_loop() if !$state;
+                                   });
+
+       chgstate('');
+       $nlcount = 0;
+       
+       return $ser;
+}
 
+sub doclose
+{
+       return if $closing++;
+       
+       dbg "serial port closing" if $ser || $ob;
+       if ($ser) {
+               $ser->stop;
+               $ser->close;
+               undef $ser;
+       }
+       if ($ob) {
+               $ob->close();
+               undef $ob;
+       }
        Mojo::IOLoop->remove($tid) if $tid;
        undef $tid;
        Mojo::IOLoop->remove($rid) if $rid;
        undef $rid;
-       $rid = Mojo::IOLoop->recurring(2.5 => sub {
-                                                                          start_loop() if !$state;
-                                                                  });
+
+    if (Mojo::IOLoop->is_running && $ending == 0) {
+               Mojo::IOLoop->delay(
+                                                       sub {
+                                                               my $delay = shift;
+                                                               Mojo::IOLoop->timer(5 => $delay->begin);
+                                                               dbg "Waiting 5 seconds before opening serial port";
+                                                        },
+
+                                                       sub {
+                                                               dbg "Opening Serial port";
+                                                               $ser = doopen($devname);
+                                                               $closing = 0;
+                                                       }
+                                                  )->wait;
+       }
        chgstate('');
        $nlcount = 0;
-       
-       return $str;
 }
 
 my @min;
@@ -218,78 +338,123 @@ sub process
        }
 
        my $tmp;
+       my $temp;
        my $rain;
        my %h;
 
-       # Common ones
-       $tmp = unpack("s", substr $blk,7,2) / 1000;
-       $h{Pressure} = sprintf("%.0f",in2mb($tmp))+0;
-
-       $tmp = unpack("s", substr $blk,9,2) / 10;
-       $h{Temp_In} = sprintf("%.1f", f2c($tmp))+0;
-       
-       $tmp  = unpack("s", substr $blk,12,2) / 10;
-       $h{Temp_Out}  = sprintf("%.1f", f2c($tmp))+0;
-
-       $tmp = unpack("C", substr $blk,14,1);
-       $h{Wind} = sprintf("%.1f",mph2mps($tmp))+0;
-       $h{Dir}     = unpack("s", substr $blk,16,2)+0;
+       my $crc_calc = CRC_CCITT($blk);
 
-       my $wind = {w => $h{Wind}, d => $h{Dir}};
-       push @min, $wind;
+       if ($crc_calc==0) {
 
-       $h{Humidity_Out} = unpack("C", substr $blk,33,1)+0;
-       $h{Humidity_In}  = unpack("C", substr $blk,11,1)+0;
+               # Common ones
+               $tmp = unpack("s", substr $blk,7,2) / 1000;
+               $h{Pressure} = nearest(0.1, in2mb($tmp));
 
-       $tmp = unpack("C", substr $blk,43,1)+0;
-       $h{UV} = $tmp unless $tmp >= 255;
-       $tmp = unpack("s", substr $blk,44,2)+0; # watt/m**2
-       $h{Solar} = $tmp unless $tmp >= 32767;
+               $tmp = unpack("s", substr $blk,9,2) / 10;
+               $h{Temp_In} = nearest(0.1, f2c($tmp));
 
-#      $h{Rain_Rate}  = sprintf("%0.1f",unpack("s", substr $blk,41,2) * $rain_mult)+0;
-       $rain = $h{Rain_Day}   = sprintf("%0.1f", unpack("s", substr $blk,50,2) * $rain_mult)+0;
-       $h{Rain} = ($rain >= $last_rain ? $rain - $last_rain : $rain) if $loop_count;
-       $last_rain = $rain;
+               $temp  = nearest(0.1, f2c(unpack("s", substr $blk,12,2) / 10));
+               $h{Temp_Out}  = $temp;
+               if ($temp > 60 || $temp < -60) {
+                       dbg "LOOP Temperature out of range ($temp), record ignored";
+                       return;
+               }
 
-       # what sort of packet is it?
+               $tmp = unpack("C", substr $blk,14,1);
+               $h{Wind} = nearest(0.1, mph2mps($tmp));
+               $h{Dir}     = unpack("s", substr $blk,16,2)+0;
 
-       my $sort =  unpack("C", substr $blk,4,1);
-       if ($sort) {
+               my $wind = {w => $h{Wind}, d => $h{Dir}};
+               $wind = 0 if $wind == 255;
+               push @{$ld->{wind_min}}, $wind;
 
-               # Newer LOOP2 packet
-               $tmp = unpack("C", substr $blk,18,2);
-#              $h{Wind_Avg_10} = sprintf("%.1f",mph2mps($tmp/10))+0;
-               $tmp = unpack("C", substr $blk,20,2);
-#              $h{Wind_Avg_2} = sprintf("%.1f",mph2mps($tmp/10))+0;
-               $tmp = unpack("C", substr $blk,22,2);
-#              $h{Wind_Gust_10} = sprintf("%.1f",mph2mps($tmp/10))+0;
+               $tmp = int(unpack("C", substr $blk,33,1)+0);
+               if ($tmp > 100) {
+                       dbg "LOOP Outside Humidity out of range ($tmp), record ignored";
+                       return;
+               }
+               $h{Humidity_Out} = $tmp;
+               $tmp = int(unpack("C", substr $blk,11,1)+0);
+               if ($tmp > 100) {
+                       dbg "LOOP Inside Humidity out of range ($tmp), record ignored";
+                       return;
+               }
+               $h{Humidity_In}  = $tmp;
+               
+
+               $tmp = unpack("C", substr $blk,43,1)+0;
+               $h{UV} = $tmp unless $tmp >= 255;
+               $tmp = unpack("s", substr $blk,44,2)+0; # watt/m**2
+               $h{Solar} = $tmp unless $tmp >= 32767;
+
+               #       $h{Rain_Rate}  = nearest(0.1,unpack("s", substr $blk,41,2) * $rain_mult);
+               $rain = $h{Rain_Day}   = nearest(0.1, unpack("s", substr $blk,50,2) * $rain_mult);
+               my $delta_rain = $h{Rain} = nearest(0.1, ($rain >= $ld->{last_rain} ? $rain - $ld->{last_rain} : $rain)) if $loop_count;
+               $ld->{last_rain} = $rain;
+
+               # what sort of packet is it?
+               my $sort =  unpack("C", substr $blk,4,1);
+               if ($sort) {
+
+                       # Newer LOOP2 packet
+                       $tmp = unpack("C", substr $blk,18,2);
+                       #               $h{Wind_Avg_10} = nearest(0.1,mph2mps($tmp/10));
+                       $tmp = unpack("C", substr $blk,20,2);
+                       #               $h{Wind_Avg_2} = nearest(0.1,mph2mps($tmp/10));
+                       $tmp = unpack("C", substr $blk,22,2);
+                       #               $h{Wind_Gust_10} = nearest(0.1,mph2mps($tmp/10));
+
+                       #               $h{Dir_Avg_10} = unpack("C", substr $blk,24,2)+0;
+                       $tmp = unpack("C", substr $blk,30,2);
+                       $h{Dew_Point} = nearest(0.1, f2c($tmp));
 
-#              $h{Dir_Avg_10} = unpack("C", substr $blk,24,2)+0;
-               $tmp = unpack("C", substr $blk,30,2);
-               $h{Dew_Point} = sprintf("%0.1f", f2c($tmp))+0;
+               } else {
 
-       } else {
+                       # Older LOOP packet
+                       $tmp = unpack("C", substr $blk,15,1);
+                       #               $h{Wind_Avg_10} = nearest(0.1,mph2mps($tmp));
+                       $h{Dew_Point}  = nearest(0.1, dew_point($h{Temp_Out}, $h{Humidity_Out}));
+                       $h{Rain_Month}  = nearest(0.1, unpack("s", substr $blk,52,2) * $rain_mult);
+                       $h{Rain_Year}  = nearest(0.1, unpack("s", substr $blk,54,2) * $rain_mult);
+               }
 
-               # Older LOOP packet
-               $tmp = unpack("C", substr $blk,15,1);
-#              $h{Wind_Avg_10} = sprintf("%.1f",mph2mps($tmp))+0;
-               $h{Dew_Point}  = sprintf("%0.1f", dew_point($h{Temp_Out}, $h{Humidity_Out}))+0;
-               $h{Rain_Month}  = sprintf("%0.1f", unpack("s", substr $blk,52,2) * $rain_mult)+0;
-               $h{Rain_Year}  = sprintf("%0.1f", unpack("s", substr $blk,54,2) * $rain_mult)+0;
+               my $ts = time;
+               my $s;
+               my $dayno = int($ts/86400);
+               my $writeld;
+               my $cycledata;
+               
+               if ($dayno > $ld->{last_day}) {
+                       $ld->{Wind_Max} = $wind->{w};
+                       $ld->{Temp_Out_Max} = $ld->{Temp_Out_Min} = $temp;
+                       $ld->{Temp_Out_Max_T} = $ld->{Temp_Out_Min_T} = $ld->{Wind_Max_T} = clocktime($ts, 0);
+                       $ld->{last_day} = $dayno;
+                       ++$writeld;
+                       ++$cycledata;
+               }
+               if ($temp > $ld->{Temp_Out_Max}) {
+                       $h{Temp_Out_Max} = $ld->{Temp_Out_Max} = $temp;
+                       $h{Temp_Out_Max_T} = $ld->{Temp_Out_Max_T} = clocktime($ts, 0);
+                       ++$writeld;
+               }
+               if ($temp < $ld->{Temp_Out_Min}) {
+                       $h{Temp_Out_Min} = $ld->{Temp_Out_Min} = $temp;
+                       $h{Temp_Out_Min_T} = $ld->{Temp_Out_Min_T} = clocktime($ts, 0);
+                       ++$writeld;
+               }
 
-       }
+               if ($wind->{w} > $ld->{Wind_Max}) {
+                       $h{Wind_Max} = $ld->{Wind_Max} = $wind->{w};
+                       $h{Wind_Max_T} = $ld->{Wind_Max_T} = clocktime($ts, 0);
+                       ++$writeld;
+               }
 
-       my $crc_calc = CRC_CCITT($blk);
 
-       if ($crc_calc==0) {
-               my $ts = time;
-               my $j;
-               my $str;
-               if ($ts >= $last_hour + 3600) {
+               if ($ts >= $ld->{last_hour} + 1800) {
                        $h{Pressure_Trend}    = unpack("C", substr $blk,3,1);
                        $h{Pressure_Trend_txt} = $bar_trend{$h{Pressure_Trend}};
                        $h{Batt_TX_OK}  = (unpack("C", substr $blk,86,1)+0) ^ 1;
-                       $h{Batt_Console}  = sprintf("%0.2f", unpack("s", substr $blk,87,2) * 0.005859375)+0;
+                       $h{Batt_Console}  = nearest(0.01, unpack("s", substr $blk,87,2) * 0.005859375);
                        $h{Forecast_Icon}  = unpack("C", substr $blk,89,1);
                        $h{Forecast_Rule}  = unpack("C", substr $blk,90,1);
                        $h{Sunrise}  = sprintf( "%04d", unpack("S", substr $blk,91,2) );
@@ -298,53 +463,90 @@ sub process
                        $h{Sunset}  =~ s/(\d{2})(\d{2})/$1:$2/;
 
                        if ($loop_count) {      # i.e not the first
-                               my $a = average(scalar @hour ? @hour : {w => $h{Wind}, d => $h{Dir}});
+                               my $a = wind_average(scalar @{$ld->{wind_hour}} ? @{$ld->{wind_hour}} : {w => $h{Wind}, d => $h{Dir}});
+
+                               $h{Wind_1h} = nearest(0.1, $a->{w});
+                               $h{Dir_1h} = nearest(0.1, $a->{d});
 
-                               $h{Wind_1h} = sprintf("%0.1f", $a->{w})+0;
-                               $h{Dir_1h} = sprintf("%0.0f", $a->{d})+0;
-                               $h{Rain_1h} = $rain >= $last_rain_hour ? $rain - $last_rain_hour : $rain;
+                               $a = wind_average(@{$ld->{wind_min}});
+                               $h{Wind_1m} = nearest(0.1, $a->{w});
+                               $h{Dir_1m} = nearest(1, $a->{d});
 
-                               $h{Wind_1m} = sprintf("%0.1f", $a->{w})+0;
-                               $h{Dir_1m} = sprintf("%0.0f", $a->{d})+0;
-                               $h{Rain_1m} = $rain >= $last_rain_min ? $rain - $last_rain_min : $rain;
+                               ($h{Rain_1m}, $h{Rain_1h}, $h{Rain_24h}) = calc_rain($rain);
+                               
                        }
-                       $last_rain_min = $last_rain_hour = $rain;
-
-                       $j = $json->encode(\%h);
-                       $str = qq|{"t":$ts,"h":$j}|;
-                       $last_hour = int($ts/3600)*3600;
-                       $last_min = int($ts/60)*60;
-                       @hour = ();
-                       @min = ();
-               } elsif ($ts >= $last_min + 60) {
-                       my $a = average(@min);
+                       $ld->{last_rain_min} = $ld->{last_rain_hour} = $rain;
+                       $h{Temp_Out_Max} = $ld->{Temp_Out_Max};
+                       $h{Temp_Out_Max_T} = $ld->{Temp_Out_Max_T};
+                       $h{Temp_Out_Min} = $ld->{Temp_Out_Min};
+                       $h{Temp_Out_Min_T} = $ld->{Temp_Out_Min_T};
+                       $h{Wind_Max} = $ld->{Wind_Max};
+                       $h{Wind_Max_T} = $ld->{Wind_Max_T};
+
+                       $last_hour_h = {%h};
+                       $s = genstr($ts, 'h', \%h);
+                       $ld->{lasthour_h} = $s;
+                       
+                       $ld->{last_hour} = int($ts/1800)*1800;
+                       $ld->{last_min} = int($ts/60)*60;
+                       @{$ld->{wind_hour}} = ();
+                       @{$ld->{wind_min}} = ();
+
+                       if ($s) {
+                               output_str($s, 1);
+                               push @last5daysh, $s;
+                               shift @last5daysh if @last5daysh > 5*24;
+                       }
+                       ++$writeld;
+                       
+               } elsif ($ts >= $ld->{last_min} + 60) {
+                       my $a = wind_average(@{$ld->{wind_min}});
                        my %save;
 
-                       push @hour, $a;
+                       push @{$ld->{wind_hour}}, $a;
 
                        if ($loop_count) {      # i.e not the first
-                               $h{Wind_1m} = sprintf("%0.1f", $a->{w})+0;
-                               $h{Dir_1m} = sprintf("%0.0f", $a->{d})+0;
-                               $h{Rain_1h} = $rain >= $last_rain_hour ? $rain - $last_rain_hour : $rain; # this is the rate for this hour, so far
-                               $h{Rain_1m} = $rain >= $last_rain_min ? $rain - $last_rain_min : $rain;
+                               my $rm;
+                               
+                               $h{Wind_1m} = nearest(0.1, $a->{w});
+                               $h{Dir_1m} = nearest(1, $a->{d});
+                               ($h{Rain_1m}, $h{Rain_1h}, $h{Rain_24h}) = calc_rain($rain);
+
+                               my $wkph = $a->{w} * 3.6;
+                               $h{WindChill} = nearest(0.1, $a->{w} >= 1.2 ? 13.12 + 0.6215 * $temp - 11.37 * $wkph ** 0.16 + 0.3965 * $temp * $wkph ** 0.16 : $temp); 
                        }
-                       $last_rain_min = $rain;
+                       $ld->{last_rain_min} = $rain;
+                       $h{Temp_Out_Max} = $ld->{Temp_Out_Max};
+                       $h{Temp_Out_Max_T} = $ld->{Temp_Out_Max_T};
+                       $h{Temp_Out_Min} = $ld->{Temp_Out_Min};
+                       $h{Temp_Out_Min_T} = $ld->{Temp_Out_Min_T};
+                       $h{Wind_Max} = $ld->{Wind_Max};
+                       $h{Wind_Max_T} = $ld->{Wind_Max_T};
+
+                       $last_min_h = {%h};
+                       $s = genstr($ts, 'm', \%h);
+                       $ld->{lastmin_h} = $s;
+                       
+                       $ld->{last_min} = int($ts/60)*60;
+                       @{$ld->{wind_min}} = ();
+                       
+                       output_str($s, 1) if $s;
+                       ++$writeld;
 
-                       $j = $json->encode(\%h);
-                       $str = qq|{"t":$ts,"m":$j}|;
-                       $last_min = int($ts/60)*60;
-                       @min = ();
                } else {
-                       my $o = gen_hash_diff($last_reading, \%h);
-                       if ($o) {
-                               $j = $json->encode($o);
-                               $str = qq|{"t":$ts,"r":$j}|;
-                       } else {
-                               dbg "loop rec not changed" if isdbg 'chan';
-                       }
+                       my $o = gen_hash_diff($ld->{last_h}, \%h);
+                       $o ||= {};
+                       # we always send wind even if it hasn't changed in order to update the wind rose.
+                       $o->{Dir} ||= ($h{Dir} + 0);
+                       $o->{Wind} ||= ($h{Wind} + 0);
+                       $s = genstr($ts, 'r', $o);
+                       push @last10minsr, $s;
+                       shift @last10minsr while @last10minsr > ($windmins * $updatepermin);
+                       output_str($s, 0) if $s;
                }
-               output_str($str) if $str;
-               $last_reading = \%h;
+               $ld->{last_h} = \%h;
+               write_ld() if $writeld;
+               cycle_loop_data_files() if $cycledata;
                ++$loop_count;
        } else {
                dbg "CRC check failed for LOOP data!";
@@ -352,12 +554,47 @@ sub process
        return;
 }
 
+sub genstr
+{
+       my $ts = shift;
+       my $let = shift;
+       my $h = shift;
+       
+       my $j =  $json->encode($h);
+       my $tm = clocktime($ts, 1);
+       return qq|{"tm":"$tm","t":$ts,"$let":$j}|;
+}
+
+sub clocktime
+{
+       my $ts = shift;
+       my $secsreq = shift;
+       my ($sec,$min,$hr) = (gmtime $ts)[0,1,2];
+       my $s;
+       if ($secsreq) {
+               $s = sprintf "%02d:%02d:%02d", $hr, $min, $sec;
+       } else {
+               $s = sprintf "%02d:%02d", $hr, $min;
+       }
+       return $s;
+}
+
 sub output_str
 {
-       my $str = shift;
-       dbg $str;
-#      say $str;
-       $dlog->writenow($str);
+       my $s = shift;
+       my $logit = shift;
+       
+       dbg $s;
+#      say $s;
+       $dlog->writenow($s) if $logit;
+       foreach my $ws (keys $WS) {
+               my $tx = $WS->{$ws};
+               if ($tx) {
+                       $tx->send($s);
+               } else {
+                       delete $WS->{$tx};
+               }
+       }
 }
 
 sub gen_hash_diff
@@ -368,7 +605,7 @@ sub gen_hash_diff
        my $count;
 
        while (my ($k, $v) = each %$now) {
-               if ($last->{$k} ne $now->{$k}) {
+               if (!exists $last->{$k} || $last->{$k} ne $now->{$k}) {
                        $o{$k} = $v;
                        ++$count;
                }
@@ -435,20 +672,129 @@ sub in2mb
        return $_[0] * 33.8637526;
 }
 
-sub average
+sub wind_average
 {
-       my %out;
+       my ($sindir, $cosdir, $wind);
        my $count;
        
        foreach my $r (@_) {
-               while (my ($k, $v) = each %$r) {
-                       $out{$k} += $v;
-               }
+               $wind += $r->{w};
+               $sindir += sin(d2r($r->{d})) * $r->{w};
+               $cosdir += cos(d2r($r->{d})) * $r->{w};
                ++$count;
        }
-       while (my ($k, $v) = each %out) {
-               $out{$k} /= $count;
+
+       my $avhdg = r2d(atan2($sindir, $cosdir));
+       $avhdg += 360 if $avhdg < 0; 
+       return {w => nearest(0.1,$wind / $count), d => nearest(0.1,$avhdg)};
+}
+
+# radians to degrees
+sub r2d
+{
+    my $n = shift;
+    return ($n / pi) * 180;
+}
+
+# degrees to radians
+sub d2r 
+{
+    my $n = shift;
+    return ($n / 180) * pi;
+}
+
+sub calc_rain
+{
+       my $rain = shift;
+       
+       $ld->{rain24} ||= [];
+       
+       my $Rain_1h = nearest(0.1, $rain >= $ld->{last_rain_hour} ? $rain - $ld->{last_rain_hour} : $rain); # this is the rate for this hour, so far
+       my $rm = nearest(0.1, $rain >= $ld->{last_rain_min} ? $rain - $ld->{last_rain_min} : $rain);
+       my $Rain_1m = nearest(0.1, $rm);
+       push @{$ld->{rain24}}, $Rain_1m;
+       $ld->{rain_24} += $rm;
+       while (@{$ld->{rain24}} > 24*60) {
+               $ld->{rain_24} -= shift @{$ld->{rain24}};
+       }
+       my $Rain_24h = nearest(0.1, $ld->{rain_24});
+       return ($Rain_1m, $Rain_1h, $Rain_24h);
+}
+
+sub read_ld
+{
+       unless ($dataf) {
+               $dataf = IO::File->new("+>> $datafn") or die "cannot open $datafn $!";
+               $dataf->autoflush(1);
        }
 
-       return \%out;
+       seek $dataf, 0, 0;
+       my $s = <$dataf>;
+       chomp $s;
+       dbg "read loop data: $s" if isdbg 'json';
+       $ld = $json->decode($s) if length $s;
+       
+       # sort out rain stats
+       my $c;
+       if ($ld->{rain24} && ($c = @{$ld->{rain24}}) < 24*60) {
+               my $diff = 24*60 - $c;
+               unshift @{$ld->{rain24}}, 0 for 0 .. $diff;  
+       }
+       my $rain;
+       
+       if ($ld->{rain24}) {
+               $rain += $_ for @{$ld->{rain24}};
+       }
+       
+       $ld->{rain_24} = nearest(0.1, $rain);
+       delete $ld->{hour};
+       delete $ld->{min};
+}
+
+sub write_ld
+{
+       unless ($dataf) {
+               $dataf = IO::File->new("+>> $datafn") or die "cannot open $datafn $!";
+               $dataf->autoflush(1);
+       }
+
+       seek $dataf, 0, 0;
+       truncate $dataf, 0;
+       $ld->{ts} = time;
+       my $s = $json->encode($ld);
+       dbg "write loop data: $s" if isdbg 'json';
+       print $dataf "$s\n";
+}
+
+sub cycle_loop_data_files
+{
+       $dataf->close if $dataf;
+       undef $dataf;
+       
+       rename "$datafn.oooo", "$datafn.ooooo";
+       rename "$datafn.ooo", "$datafn.oooo";
+       rename "$datafn.oo", "$datafn.ooo";
+       rename "$datafn.o", "$datafn.oo";
+       copy $datafn, "$datafn.o";
+}
+
+sub grab_history
+{
+       my $lg = shift;
+       my $let = shift;
+       my $start = shift || time - 86400;
+       my $dayno = shift;
+       my @out;
+       
+       if ($lg->open($dayno, 'r+')) {
+               while (my $l = $lg->read) {
+                       next unless $l =~ /,"$let":/;
+                       my ($t) = $l =~ /"t":(\d+)/;
+                       if ($t && $t >= $start) {
+                               push @out, $l;
+                       }
+               }
+               $lg->close;
+       }
+       return @out;
 }