X-Git-Url: http://gb7djk.dxcluster.net/gitweb/gitweb.cgi?a=blobdiff_plain;ds=sidebyside;f=perl%2FDXUser.pm;h=fb88ad824f606f317036ada284d83617f19d3775;hb=bfd4e140dcf03f12bbda1dd405f8b135f86d0b6a;hp=d7c6a1ae73aac90d476f36bcf6bd7416bbbd6fe7;hpb=e67d75717f0625225632cfd12a7a2d899fb692ea;p=spider.git diff --git a/perl/DXUser.pm b/perl/DXUser.pm index d7c6a1ae..fb88ad82 100644 --- a/perl/DXUser.pm +++ b/perl/DXUser.pm @@ -75,10 +75,9 @@ use IO::File; use strict; -use vars qw(%u $dbm $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $v3 $v4); +use vars qw(%u $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $v3 $v4); %u = (); -$dbm = undef; $filename = undef; $lastoperinterval = 60*24*60*60; $lasttime = 0; @@ -90,10 +89,11 @@ my $json; our $maxconnlist = 3; # remember this many connection time (duration) [start, end] pairs -our $newusers; # per execution stats -our $modusers; -our $totusers; -our $delusers; +our $newusers = 0; # per execution stats +our $modusers = 0; +our $totusers = 0; +our $delusers = 0; +our $cachedusers = 0; my $ifh; # the input file, initialised by readinjson() @@ -187,92 +187,35 @@ sub init { my $mode = shift; - my $ufn; - my $convert; - - my $fn = "users"; - + my $convert = "$main::root/perl/convert-users-v3-to-v4.pl"; + my $export; + $json = JSON->new()->canonical(1); - $filename = $ufn = localdata("$fn.json"); + $filename = localdata("users.v4"); - if (-e localdata("$fn.json")) { + if (-e $filename || -e "$filename.n" || -e "$filename.o") { $v4 = 1; } else { - eval { - require Storable; - }; - - if ($@) { - if ( ! -e localdata("users.v3") && -e localdata("users.v2") ) { - $convert = 2; - } - LogDbg('',"the module Storable appears to be missing!!"); - LogDbg('',"trying to continue in compatibility mode (this may fail)"); - LogDbg('',"please install Storable from CPAN as soon as possible"); - } else { - import Storable qw(nfreeze thaw); - $convert = 3 if -e localdata("users.v3") && !-e $ufn; - } - } - - # do a conversion if required - if ($convert) { - my ($key, $val, $action, $count, $err) = ('','',0,0,0); - my $ta = [gettimeofday]; +# if (-e localdata('users.v3')) { +# LogDbg('DXUser', "Converting " . localdata('users.v3') . " to new json version of users file, please wait"); +# if (-x $convert) { +# system($convert); +# ++$export; +# } +# } - my %oldu; - LogDbg('',"Converting the User File from V$convert to $fn.json "); - LogDbg('',"This will take a while, I suggest you go and have cup of strong tea"); - my $odbm = tie (%oldu, 'DB_File', localdata("users.v$convert"), O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn.v$convert ($!) [rebuild it from user_asc?]"; - for ($action = R_FIRST; !$odbm->seq($key, $val, $action); $action = R_NEXT) { - my $ref; - if ($convert == 3) { - eval { $ref = storable_decode($val) }; - } else { - eval { $ref = asc_decode($val) }; - } - unless ($@) { - if ($ref) { - $u{$key} = $ref; - $count++; - } else { - $err++ - } - } else { - Log('err', "DXUser: error decoding $@"); - } - } - undef $odbm; - untie %oldu; - my $t = _diffms($ta); - LogDbg('',"Conversion from users.v$convert to users.json completed $count records $err errors $t mS"); - - # now write it away for future use - $ta = [gettimeofday]; - $err = 0; - $count = writeoutjson(); - $t = _diffms($ta); - LogDbg('',"New Userfile users.json write completed $count records $err errors $t mS"); - LogDbg('',"Now restarting.."); - $main::ending = 10; - } else { - # otherwise (i.e normally) slurp it in - readinjson(); + die "User file $filename missing, please run $convert or copy a user_json backup from somewhere\n" unless -e "$filename.n" || -s $filename; } - $filename = $ufn; + readinjson(); + copy $filename, "$filename.n" unless -e "$filename.n"; + export() if $export; } sub del_file { # with extreme prejudice - if ($v3) { - unlink "$main::data/users.v3"; - unlink "$main::local_data/users.v3"; - } - if ($v4) { - unlink "$main::data/users.v4"; - unlink "$main::local_data/users.v4"; - } + unlink "$main::data/users.v4"; + unlink "$main::local_data/users.v4"; } # @@ -292,8 +235,8 @@ sub process sub finish { - undef $dbm; - untie %u; + + writeoutjson(); } # @@ -314,9 +257,10 @@ sub new my $call = shift; # $call =~ s/-\d+$//o; -# confess "can't create existing call $call in User\n!" if $u{$call}; + confess "can't create existing call $call in User\n!" if $u{$call}; my $self = $pkg->alloc($call); + $u{$call} = [0, $self]; $self->put; ++$newusers; ++$totusers; @@ -344,6 +288,7 @@ sub get my $j = json_decode($s); if ($j) { $ref->[1] = $j; + ++$cachedusers; } } } elsif ($nodecode) { @@ -359,44 +304,61 @@ sub get # This is not as quick as get()! But it will allow safe querying of the # user file. Probably in conjunction with get_some_calls feeding it. # -# Probably need to create a new copy of any existing records WIP +# NOTE: for cached records this, in effect, is a faster version of Storable's +# dclone - only about 3.5 times as fast! +# sub get_tmp { my $call = uc shift; - my $ref = $u{call}; + my $ref = $u{$call}; if ($ref) { + if ($ref->[1]) { + return json_decode(json_encode($ref->[1])); + } $ifh->seek($ref->[0], 0); my $l = $ifh->getline; if ($l) { my ($k,$s) = split /\t/, $l; my $j = json_decode($s); - return $; + return $j; } } return undef; } # -# get an existing either from the channel (if there is one) or from the database +# Master branch: +# get an existing record either from the channel (if there is one) or from the database # # It is important to note that if you have done a get (for the channel say) and you # want access or modify that you must use this call (and you must NOT use get's all # over the place willy nilly!) # +# NOTE: mojo branch with newusers system: +# There is no longer any function difference between get_current() +# and get() as they will always reference the same record as held in %u. This is because +# there is no more (repeated) thawing of stored records from the underlying "database". +# +# BUT: notice the difference between this and the get_tmp() function. A get() will online an +# othewise unused record, so for poking around looking for that locked out user: +# MAKE SURE you use get_tmp(). It will likely still be quicker than DB_File and Storable! +# sub get_current { - my $call = uc shift; - - my $dxchan = DXChannel::get($call); - if ($dxchan) { - my $ref = $dxchan->user; - return $ref if $ref && UNIVERSAL::isa($ref, 'DXUser'); - - dbg("DXUser::get_current: got invalid user ref for $call from dxchan $dxchan->{call} ". ref $ref. " ignoring"); - } - return get($call); + goto &get; + +# my $call = uc shift; +# +# my $dxchan = DXChannel::get($call); +# if ($dxchan) { +# my $ref = $dxchan->user; +# return $ref if $ref && UNIVERSAL::isa($ref, 'DXUser'); +# +# dbg("DXUser::get_current: got invalid user ref for $call from dxchan $dxchan->{call} ". ref $ref. " ignoring"); +# } +# return get($call); } # @@ -417,7 +379,40 @@ sub get_some_calls my $pattern = shift || qr/.*/; return sort grep {$pattern} keys %u; } + +# +# if I understand the term correctly, this is a sort of monad. +# +# Scan through the whole user file and select records that you want +# to process further. This routine returns lines of json, yu +# +# the CODE ref should look like: +# sub { +# my $key = shift; +# my $line = shift; +# # maybe do a rough check to see if this is a likely candidate +# return unless $line =~ /something/; +# my $r = json_decode($l); +# return (condition ? wanted thing : ()); +# } +# + +sub scan +{ + my $c = shift; + my @out; + if (ref($c) eq 'CODE') { + foreach my $k (get_all_calls()) { + my $l = get($k, 1); # get the offline json line or the jsoned online version + push @out, $c->($k, $l) if $l; + } + } else { + dbg("DXUser::scan supplied argument is not a code ref"); + } + return @out; +} + # # put - put a user # @@ -433,43 +428,13 @@ sub put # freeze the user sub encode { - goto &json_encode if $v4; - goto &asc_encode unless $v3; - my $self = shift; - return nfreeze($self); + goto &json_encode; } # thaw the user sub decode { - goto &json_decode if $v4; - goto &storable_decode if $v3; - goto &asc_decode; -} - -# should now be obsolete for mojo branch build 238 and above -sub storable_decode -{ - my $ref; - $ref = thaw(shift); - return $ref; -} - - -# -# create a hash from a string (in ascii) -# -sub asc_decode -{ - my $s = shift; - my $ref; - $s =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg; - eval '$ref = ' . $s; - if ($@) { - LogDbg('err', "DXUser::asc_decode: on '$s' $@"); - $ref = undef; - } - return $ref; + goto &json_decode; } sub json_decode @@ -480,7 +445,7 @@ sub json_decode if ($ref && !$@) { return bless $ref, 'DXUser'; } else { - LogDbg('err', "DXUser::json_decode: on '$s' $@"); + LogDbg('DXUser', "DXUser::json_decode: on '$s' $@"); } return undef; } @@ -502,10 +467,9 @@ sub del { my $self = shift; my $call = $self->{call}; -# $lru->remove($call); - # $dbm->del($call); ++$delusers; --$totusers; + --$cachedusers if $u{$call}->[1]; delete $u{$call}; } @@ -524,7 +488,6 @@ sub close push @$ref, $ip if $ip; push @{$self->{connlist}}, $ref; shift @{$self->{connlist}} if @{$self->{connlist}} > $maxconnlist; -# $self->put(); } # @@ -555,6 +518,7 @@ sub export my $name = shift; my $fn = $name || localdata("user_json"); # force use of local_data + my $ta = [gettimeofday]; # save old ones move "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo"; @@ -564,7 +528,7 @@ sub export move "$fn", "$fn.o" if -e "$fn"; my $json = JSON->new; - $json->canonical(1); + $json->canonical(1);; my $count = 0; my $err = 0; @@ -574,10 +538,10 @@ sub export my $key = 0; my $val = undef; foreach my $k (sort keys %u) { - my $r = $u{$k}; + my $r = get($k); if ($r->{sort} eq 'U' && !$r->{priv} && $main::systime > $r->{lastin}+$tooold ) { unless ($r->{lat} || $r->{long} || $r->{qra} || $r->{qth} || $r->{name}) { - LogDbg('err', "DXUser::export deleting $k - too old, last in " . cldatetime($r->lastin) . " " . difft([$r->lastin, $main::systime])); + LogDbg('export', "DXUser::export deleting $k - too old, last in " . cldatetime($r->lastin) . " " . difft([$r->lastin, $main::systime])); delete $u{$k}; ++$del; next; @@ -585,7 +549,7 @@ sub export } eval {$val = json_encode($r);}; if ($@) { - LogDbg('err', "DXUser::export error encoding call: $k $@"); + LogDbg('export', "DXUser::export error encoding call: $k $@"); ++$err; next; } @@ -594,8 +558,9 @@ sub export } $fh->close; } - my $s = qq{Exported users to $fn - $count Users $del Deleted $err Errors ('sh/log Export' for details)}; - LogDbg('command', $s); + my $t = _diffms($ta); + my $s = qq{Exported users to $fn - $count Users $del Deleted $err Errors in $t mS ('sh/log Export' for details)}; + LogDbg('DXUser', $s); return $s; } @@ -875,12 +840,14 @@ sub unset_passwd { my $self = shift; delete $self->{passwd}; + $self->put; } sub unset_passphrase { my $self = shift; delete $self->{passphrase}; + $self->put; } sub set_believe @@ -888,7 +855,10 @@ sub set_believe my $self = shift; my $call = uc shift; $self->{believe} ||= []; - push @{$self->{believe}}, $call unless grep $_ eq $call, @{$self->{believe}}; + unless (grep $_ eq $call, @{$self->{believe}}) { + push @{$self->{believe}}, $call; + $self->put; + }; } sub unset_believe @@ -898,6 +868,7 @@ sub unset_believe if (exists $self->{believe}) { $self->{believe} = [grep {$_ ne $call} @{$self->{believe}}]; delete $self->{believe} unless @{$self->{believe}}; + $self->put; } } @@ -923,8 +894,6 @@ sub lastping # a later (generated) copy. But, if the plain users.v4 file is all we have, we'll use that. # -use File::Copy; - sub readinjson { my $fn = $filename; @@ -936,11 +905,6 @@ sub readinjson my $s; my $err = 0; - unless (-r $fn) { - dbg("DXUser $fn not found - probably about to convert"); - return; - } - if (-e $nfn && -e $fn && (stat($nfn))[9] > (stat($fn))[9]) { # move the old file to .o unlink $ofn; @@ -948,10 +912,18 @@ sub readinjson move($nfn, $fn); }; + # if we don't have a users.v4 at this point, look for a backup users.v4.json, users.v4.n then users.v4.o + unless (-e $fn) { + move($nfn, $fn) unless -e $fn; # the users.v4 isn't there (maybe convert-users-v3-to-v4.pl + move("$fn.json", $fn); # from a run of convert-users-v3-to-v4.pl + move($ofn, $fn) unless -e $fn; # desperate now... + } + if ($ifh) { $ifh->seek(0, 0); } else { - $ifh = IO::File->new("+<$fn") or die "$fn read error $!"; + LogDbg("DXUser","DXUser::readinjson: opening $fn as users file"); + $ifh = IO::File->new("+<$fn") or die "Cannot open $fn ($!)"; } my $pos = $ifh->tell; while (<$ifh>) { @@ -965,7 +937,7 @@ sub readinjson # $ifh is "global" and should not be closed - dbg("DXUser::readinjson $count record headers read from $fn in ". _diffms($ta) . " mS"); + LogDbg('DXUser',"DXUser::readinjson $count record headers read from $fn in ". _diffms($ta) . " mS"); return $totusers = $count; } @@ -989,12 +961,12 @@ sub writeoutjson print $ofh "$k\t$l\n"; ++$count; } else { - LogDbg('DXCommand', "DXUser::writeoutjson callsign $k not found") + LogDbg('DXUser', "DXUser::writeoutjson callsign $k not found") } } $ofh->close; - dbg("DXUser::writeoutjson $count records written to $ofn in ". _diffms($ta) . " mS"); + LogDbg('DXUser',"DXUser::writeoutjson $count records written to $ofn in ". _diffms($ta) . " mS"); return $count; } 1;