X-Git-Url: http://gb7djk.dxcluster.net/gitweb/gitweb.cgi?a=blobdiff_plain;f=perl%2FDXUser.pm;h=d147d68afe29fed5aa5ecfa47a2cd24ca6348dfd;hb=7e3729a9039462c460dae4804f40c0bd506f3423;hp=1fc8dd8671de0c12bed3f416366f18eb7259b18e;hpb=4a134278a40e451ff1769c7b98d1a4f709a6b828;p=spider.git diff --git a/perl/DXUser.pm b/perl/DXUser.pm index 1fc8dd86..d147d68a 100644 --- a/perl/DXUser.pm +++ b/perl/DXUser.pm @@ -17,86 +17,123 @@ use DXDebug; use DXUtil; use LRU; use File::Copy; +use Data::Structure::Util qw(unbless); +use Time::HiRes qw(gettimeofday tv_interval); +use IO::File; +use DXJSON; use strict; -use vars qw(%u $dbm $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $v3); +use vars qw(%u $dbm $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $veryold $v3); %u = (); $dbm = undef; $filename = undef; $lastoperinterval = 60*24*60*60; $lasttime = 0; -$lrusize = 2000; -$tooold = 86400 * 365; # this marks an old user who hasn't given enough info to be useful +$lrusize = 5000; +$tooold = 86400 * 365 * 2; # this marks an old user who hasn't given enough info to be useful +$veryold = $tooold * 6; # Ancient default 12 years $v3 = 0; our $maxconnlist = 3; # remember this many connection time (duration) [start, end] pairs +my $json; + # hash of valid elements and a simple prompt %valid = ( - call => '0,Callsign', - alias => '0,Real Callsign', - name => '0,Name', - qth => '0,Home QTH', - lat => '0,Latitude,slat', - long => '0,Longitude,slong', - qra => '0,Locator', - email => '0,E-mail Address,parray', - priv => '9,Privilege Level', - lastin => '0,Last Time in,cldatetime', - passwd => '9,Password,yesno', - passphrase => '9,Pass Phrase,yesno', - addr => '0,Full Address', 'sort' => '0,Type of User', # A - ak1a, U - User, S - spider cluster, B - BBS - xpert => '0,Expert Status,yesno', + K => '9,Seen on PC92 K,yesno', + addr => '0,Full Address', + addr => '0,Full Address', + alias => '0,Real Callsign', + alias => '0,Real Callsign', + annok => '9,Accept Announces?,yesno', # accept his announces? bbs => '0,Home BBS', - node => '0,Last Node', - homenode => '0,Home Node', - lockout => '9,Locked out?,yesno', # won't let them in at all + believe => '1,Believable nodes,parray', + believe => '1,Believable nodes,parray', + buddies => '0,Buddies,parray', + build => '1,Build', + build => '1,Build', + call => '0,Callsign', + call => '0,Callsign', + connlist => '1,Connections,parraydifft', dxok => '9,Accept DX Spots?,yesno', # accept his dx spots? - annok => '9,Accept Announces?,yesno', # accept his announces? - lang => '0,Language', - hmsgno => '0,Highest Msgno', + email => '0,E-mail Address,parray', + email => '0,E-mail Address,parray', group => '0,Group,parray', # used to create a group of users/nodes for some purpose or other - buddies => '0,Buddies,parray', + hmsgno => '0,Highest Msgno', + homenode => '0,Home Node', isolate => '9,Isolate network,yesno', - wantbeep => '0,Req Beep,yesno', - wantann => '0,Req Announce,yesno', - wantwwv => '0,Req WWV,yesno', - wantwcy => '0,Req WCY,yesno', - wantecho => '0,Req Echo,yesno', - wanttalk => '0,Req Talk,yesno', - wantwx => '0,Req WX,yesno', - wantdx => '0,Req DX Spots,yesno', - wantemail => '0,Req Msgs as Email,yesno', + lang => '0,Language', + lastin => '0,Last Time in,cldatetime', + lastin => '0,Last Time in,cldatetime', + lastoper => '9,Last for/oper,cldatetime', + lastoper => '9,Last for/oper,cldatetime', + lastping => '1,Last Ping at,ptimelist', + lastping => '1,Last Ping at,ptimelist', + lastseen => '0,Last Seen,cldatetime', + lat => '0,Latitude,slat', + lat => '0,Latitude,slat', + lockout => '9,Locked out?,yesno', # won't let them in at all + long => '0,Longitude,slong', + long => '0,Longitude,slong', + maxconnect => '1,Max Connections', + maxconnect => '1,Max Connections', + name => '0,Name', + name => '0,Name', + node => '0,Last Node', + nopings => '9,Ping Obs Count', + nothere => '0,Not Here Text', + nothere => '0,Not Here Text', pagelth => '0,Current Pagelth', + passphrase => '9,Pass Phrase,yesno', + passphrase => '9,Pass Phrase,yesno', + passwd => '9,Password,yesno', + passwd => '9,Password,yesno', pingint => '9,Node Ping interval', - nopings => '9,Ping Obs Count', - wantlogininfo => '0,Login Info Req,yesno', - wantgrid => '0,Show DX Grid,yesno', + priv => '9,Privilege Level', + priv => '9,Privilege Level', + prompt => '0,Required Prompt', + prompt => '0,Required Prompt', + qra => '0,Locator', + qra => '0,Locator', + qth => '0,Home QTH', + qth => '0,Home QTH', + rbnseeme => '0,RBN See Me', + registered => '9,Registered?,yesno', + registered => '9,Registered?,yesno', + startt => '0,Start Time,cldatetime', + version => '1,Version', + version => '1,Version', + wantann => '0,Req Announce,yesno', wantann_talk => '0,Talklike Anns,yesno', - wantpc16 => '9,Want Users from node,yesno', - wantsendpc16 => '9,Send PC16,yesno', - wantroutepc19 => '9,Route PC19,yesno', - wantusstate => '0,Show US State,yesno', + wantbeacon => '0,Want RBN Beacon,yesno', + wantbeep => '0,Req Beep,yesno', + wantcw => '0,Want RBN CW,yesno', + wantdx => '0,Req DX Spots,yesno', wantdxcq => '0,Show CQ Zone,yesno', wantdxitu => '0,Show ITU Zone,yesno', + wantecho => '0,Req Echo,yesno', + wantemail => '0,Req Msgs as Email,yesno', + wantft => '0,Want RBN FT4/8,yesno', wantgtk => '0,Want GTK interface,yesno', + wantlogininfo => '0,Login Info Req,yesno', + wantpc16 => '9,Want Users from node,yesno', wantpc9x => '0,Want PC9X interface,yesno', + wantpsk => '0,Want RBN PSK,yesno', wantrbn => '0,Want RBN spots,yesno', - wantft => '0,Want FT4/8 spots,yesno', - wantcw => '0,Want (RBN) CW spots,yesno', - lastoper => '9,Last for/oper,cldatetime', - nothere => '0,Not Here Text', - registered => '9,Registered?,yesno', - prompt => '0,Required Prompt', - version => '1,Version', - build => '1,Build', - believe => '1,Believable nodes,parray', - lastping => '1,Last Ping at,ptimelist', - maxconnect => '1,Max Connections', - startt => '0,Start Time,cldatetime', - connlist => '1,Connections,parraydifft', + wantroutepc19 => '9,Route PC19,yesno', + wantrtty => '0,Want RBN RTTY,yesno', + wantsendpc16 => '9,Send PC16,yesno', + wanttalk => '0,Req Talk,yesno', + wantusstate => '0,Show US State,yesno', + wantwcy => '0,Req WCY,yesno', + wantwwv => '0,Req WWV,yesno', + wantwx => '0,Req WX,yesno', + width => '0,Preferred Width', + width => '0,Preferred Width', + xpert => '0,Expert Status,yesno', + wantgrid => '0,Show DX Grid,yesno', ); #no strict; @@ -124,73 +161,36 @@ sub init { my $mode = shift; - my $ufn; - my $convert; - - eval { - require Storable; - }; - + $json = DXJSON->new->canonical(1); my $fn = "users"; - - if ($@) { - $ufn = localdata("users.v2"); - $v3 = $convert = 0; - dbg("the module Storable appears to be missing!!"); - dbg("trying to continue in compatibility mode (this may fail)"); - dbg("please install Storable from CPAN as soon as possible"); - } else { - import Storable qw(nfreeze thaw); - - $ufn = localdata("users.v3"); - $v3 = 1; - $convert++ if -e localdata("users.v2") && !-e $ufn; - } - - if ($mode) { - $dbm = tie (%u, 'DB_File', $ufn, O_CREAT|O_RDWR, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_asc?]"; - } else { - $dbm = tie (%u, 'DB_File', $ufn, O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_asc?]"; + $filename = localdata("$fn.v3j"); + unless (-e $filename || $mode == 2 ) { + if (-e localdata("$fn.v3") || -e localdata("$fn.v2")) { + LogDbg('DXUser', "New User File version $filename does not exist, running conversion from users.v3 or v2, please wait"); + system('/spider/perl/convert-users-v3-to-v3j.pl'); + init(1); + export(); + return; + } } - - die "Cannot open $ufn ($!)\n" unless $dbm; - - $lru = LRU->newbase("DXUser", $lrusize); - - # do a conversion if required - if ($dbm && $convert) { - my ($key, $val, $action, $count, $err) = ('','',0,0,0); - - my %oldu; - dbg("Converting the User File to V3 "); - dbg("This will take a while, I suggest you go and have cup of strong tea"); - my $odbm = tie (%oldu, 'DB_File', localdata("users.v2"), O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn.v2 ($!) [rebuild it from user_asc?]"; - for ($action = R_FIRST; !$odbm->seq($key, $val, $action); $action = R_NEXT) { - my $ref; - eval { $ref = asc_decode($val) }; - unless ($@) { - if ($ref) { - $ref->put; - $count++; - } else { - $err++ - } - } else { - Log('err', "DXUser: error decoding $@"); - } - } - undef $odbm; - untie %oldu; - dbg("Conversion completed $count records $err errors"); + if (-e $filename || $mode) { + $lru = LRU->newbase("DXUser", $lrusize); + if ($mode) { + $dbm = tie (%u, 'DB_File', $filename, O_CREAT|O_RDWR, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_json?]"; + } else { + $dbm = tie (%u, 'DB_File', $filename, O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_json?]"; + } } - $filename = $ufn; + die "Cannot open $filename ($!)\n" unless $dbm || $mode == 2; + return; } +# delete files with extreme prejudice sub del_file { # with extreme prejudice - unlink "$main::data/users.v3"; - unlink "$main::local_data/users.v3"; + unlink "$main::data/users.v3j"; + unlink "$main::local_data/users.v3j"; } # @@ -210,6 +210,7 @@ sub process sub finish { + $dbm->sync; undef $dbm; untie %u; } @@ -251,12 +252,13 @@ sub get # is it in the LRU cache? my $ref = $lru->get($call); - return $ref if $ref && ref $ref eq 'DXUser'; + if ($ref && ref $ref eq 'DXUser') { + return $ref; + } # search for it unless ($dbm->get($call, $data)) { eval { $ref = decode($data); }; - if ($ref) { if (!UNIVERSAL::isa($ref, 'DXUser')) { dbg("DXUser::get: got strange answer from decode of $call". ref $ref. " ignoring"); @@ -319,68 +321,27 @@ sub put my $call = $self->{call}; $dbm->del($call); - delete $self->{annok} if $self->{annok}; - delete $self->{dxok} if $self->{dxok}; - + delete $self->{annok}; + delete $self->{dxok}; + $self->{lastseen} = $main::systime; $lru->put($call, $self); my $ref = $self->encode; $dbm->put($call, $ref); } -# freeze the user -sub encode -{ - goto &asc_encode unless $v3; - my $self = shift; - return nfreeze($self); -} # thaw the user sub decode { - goto &asc_decode unless $v3; - my $ref; - $ref = thaw(shift); - return $ref; + return $json->decode(shift, __PACKAGE__); } -# -# create a string from a user reference (in_ascii) -# -sub asc_encode +# freeze the user +sub encode { - my $self = shift; - my $strip = shift; - my $p; - - if ($strip) { - my $ref = bless {}, ref $self; - foreach my $k (qw(qth lat long qra sort call homenode node lastoper lastin)) { - $ref->{$k} = $self->{$k} if exists $self->{$k}; - } - $ref->{name} = $self->{name} if exists $self->{name} && $self->{name} !~ /selfspot/i; - $p = dd($ref); - } else { - $p = dd($self); - } - return $p; + return $json->encode(shift); } -# -# create a hash from a string (in ascii) -# -sub asc_decode -{ - my $s = shift; - my $ref; - $s =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg; - eval '$ref = ' . $s; - if ($@) { - LogDbg('err', "DXUser::asc_decode: on '$s' $@"); - $ref = undef; - } - return $ref; -} # # del - delete a user @@ -403,8 +364,8 @@ sub close my $self = shift; my $startt = shift; my $ip = shift; - $self->{lastin} = $main::systime; # add a record to the connect list + $self->{lastin} = $main::systime; my $ref = [$startt || $self->{startt}, $main::systime]; push @$ref, $ip if $ip; push @{$self->{connlist}}, $ref; @@ -437,21 +398,33 @@ sub fields sub export { - my $name = shift || 'user_asc'; + my $name = shift || 'user_json'; my $basic_info_only = shift; - my $fn = $name ne 'user_asc' ? $name : "$main::local_data/$name"; # force use of local + my $fn = $name ne 'user_json' ? $name : "$main::local_data/$name"; # force use of local # save old ones + copy $fn, "$fn.keep" unless -e "$fn.keep"; + copy "$fn.ooooo", "$fn.backstop" unless -e "$fn,backstop"; + move "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo"; move "$fn.ooo", "$fn.oooo" if -e "$fn.ooo"; move "$fn.oo", "$fn.ooo" if -e "$fn.oo"; move "$fn.o", "$fn.oo" if -e "$fn.o"; move "$fn", "$fn.o" if -e "$fn"; + + my $ta = [gettimeofday]; my $count = 0; my $err = 0; my $del = 0; + my $spurious = 0; + my $unlocked = 0; + my $old = 0; + my $ancient = 0; + my $nodes = 0; + my $renamed = 0; + my $fh = new IO::File ">$fn" or return "cannot open $fn ($!)"; if ($fh) { my $key = 0; @@ -492,35 +465,39 @@ BEGIN { } use SysVar; +use DXUtil; use DXUser; +use JSON; +use Time::HiRes qw(gettimeofday tv_interval); +package DXUser; -if (@ARGV) { - $main::userfn = shift @ARGV; - print "user filename now $userfn\n"; -} +our $json = JSON->new->canonical(1); -package DXUser; +my $ta = [gettimeofday]; +our $filename = "$main::local_data/users.v3j"; +my $exists = -e $filename ? "OVERWRITING" : "CREATING"; +print "perl user_json $exists $filename\n"; del_file(); -init(1); +init(2); %u = (); my $count = 0; my $err = 0; while () { chomp; my @f = split /\t/; - my $ref = asc_decode($f[1]); + my $ref = decode($f[1]); if ($ref) { $ref->put(); $count++; - DXUser::sync() unless $count % 10000; } else { print "# Error: $f[0]\t$f[1]\n"; $err++ } } DXUser::sync(); DXUser::finish(); -print "There are $count user records and $err errors\n"; +my $diff = _diffms($ta); +print "There are $count user records and $err errors in $diff mS\n"; }; print $fh "__DATA__\n"; @@ -529,39 +506,90 @@ print "There are $count user records and $err errors\n"; my $eval = $val; my $ekey = $key; $eval =~ s/([\%\x00-\x1f\x7f-\xff])/sprintf("%%%02X", ord($1))/eg; - $ekey =~ s/([\%\x00-\x1f\x7f-\xff])/sprintf("%%%02X", ord($1))/eg; - LogDbg('DXCommand', "Export Error1: $ekey\t$eval"); + $ekey =~ s/([\%\x00-\x1f\x7f-\xff])/sprintf("%%%02X", ord($1))/eg; + LogDbg('DXCommand', "Export Error1: invalid call '$key' => '$val'"); eval {$dbm->del($key)}; - dbg(carp("Export Error1: $ekey\t$eval\n$@")) if $@; + dbg(carp("Export Error1: delete $key => '$val' $@")) if $@; ++$err; next; } my $ref; eval {$ref = decode($val); }; if ($ref) { - my $t = $ref->{lastin} || 0; - if ($ref->is_user && !$ref->{priv} && $main::systime > $t + $tooold) { - unless ($ref->{lat} && $ref->{long} || $ref->{qth} || $ref->{qra}) { + my $t = $ref->{lastseen} if exists $ref->{lastseen}; + $t ||= $ref->{lastin} if exists $ref->{lastin}; + $t ||= $ref->{lastoper} if exists $ref->{lastoper}; + $t //= 0; + + if ($ref->is_user) { + if (!$ref->{priv} && $main::systime > $t + $tooold) { + unless (($ref->{lat} && $ref->{long}) || $ref->{qth} || $ref->{name} || $ref->{qra}) { + LogDbg('DXCommand', sprintf("$ref->{call} deleted, empty and too Old at %s", difft($t, ' '))); + ++$del; + ++$old; + eval {$dbm->del($key)}; + dbg(carp("Export Error2: delete '$key' => '$val' $@")) if $@; + next; + } + } + if ($main::systime > $t + $veryold) { + LogDbg('DXCommand', sprintf("$ref->{call} deleted, POSITIVELY ANCIENT at %s", difft($t, ' '))); + ++$del; + ++$ancient; + eval {$dbm->del($key)}; + dbg(carp("Export Error2: delete '$key' => '$val' $@")) if $@; + next; + } + if (exists $ref->{lockout} && $ref->{lockout} == 1 && exists $ref->{priv} && $ref->{priv} == 1) { + LogDbg('DXCommand', "$ref->{call} depriv'd and unlocked"); + $ref->{lockout} = $ref->{priv} = 0; + $ref->put; + ++$unlocked; + } + if ($ref->is_node && $main::systime > $t + $veryold) { + LogDbg('DXCommand', sprintf("NODE $ref->{call} deleted (%s) old", difft($t, ' '))); + ++$del; + ++$nodes; + eval {$dbm->del($key)}; + dbg(carp("Export Error2: delete '$key' => '$val' $@")) if $@; + next; + } + + my $normcall = normalise_call($key); + if ($normcall ne $key) { + # if the normalised call does not exist, create it from the duff call. + my $nref = DXUser::get_current($normcall); + unless ($nref) { + $ref->{call} = $normcall; + $ref->put; + LogDbg('DXCommand', "DXProt: spurious call $key normalises to $normcall renaming $key -> $normcall"); + ++$renamed; + } + LogDbg('DXCommand', "DXProt: spurious call $key (should be $normcall), removing"); eval {$dbm->del($key)}; - dbg(carp("Export Error2: $key\t$val\n$@")) if $@; - LogDbg('DXCommand', "$ref->{call} deleted, too old"); - $del++; + dbg(carp("Export Error1: delete $key => '$val' $@")) if $@; + ++$spurious; + ++$del; next; } } - # only store users that are reasonably active or have useful information - print $fh "$key\t" . $ref->asc_encode($basic_info_only) . "\n"; - ++$count; } else { - LogDbg('DXCommand', "Export Error3: $key\t" . carp($val) ."\n$@"); + LogDbg('DXCommand', "Export Error3: '$key'\t" . carp($val) ."\n$@"); eval {$dbm->del($key)}; - dbg(carp("Export Error3: $key\t$val\n$@")) if $@; + dbg(carp("Export Error3: delete '$key' => '$val' $@")) if $@; ++$err; + next; } - } - $fh->close; - } - my $s = qq{Exported users to $fn - $count Users $del Deleted $err Errors ('sh/log Export' for details)}; + + # only store users that are reasonably active or have useful information + print $fh "$key\t" . encode($ref) . "\n"; + ++$count; + } + } + $fh->close; + + my $diff = _diffms($ta); + my $s = qq{Exported users to $fn - $count Users, $del Deleted ($old empty \& too old, $ancient ancient, $nodes nodes, $spurious spurious), $renamed renamed, $unlocked Unlocked, $err Errors in $diff mS ('sh/log Export' for details)}; LogDbg('command', $s); return $s; } @@ -717,7 +745,7 @@ sub wanttalk sub wantgrid { - return _want('grid', @_); + return _wantnot('grid', @_); } sub wantemail @@ -752,12 +780,12 @@ sub wantusstate sub wantdxcq { - return _want('dxcq', @_); + return _wantnot('dxcq', @_); } sub wantdxitu { - return _want('dxitu', @_); + return _wantnot('dxitu', @_); } sub wantgtk @@ -890,6 +918,21 @@ sub lastping $b->{$call} = shift if @_; return $b->{$call}; } + +sub registered +{ + my $self = shift; + my $val; + if (defined $_[0]) { + $val = unpad($_[0]); + $self->{registered} = $val; + } + if (exists $self->{registered}) { + $val = $self->{registered} // 0; + } + return $val // 0 ; # to stop undef warnings +} + 1; __END__