2 # DX cluster user routines
4 # Copyright (c) 1998-2020 - Dirk Koopman G1TLH
6 # The new internal structure of the users system looks like this:
8 # The users.v4 file formatted as a file of lines containing: <callsign>\t{json serialised version of user record}\n
10 # You can look at it with any text tools or your favourite editor :-)
12 # In terms of internal structure, the main user hash remains as %u, keyed on callsign as before.
14 # The value is a one or two element array [position] or [position, ref], depending on whether the record has been "get()ed"
15 # [i.e. got from disk] or not. The 'position' is simply the start of each line in the file. The function "get()" simply returns
16 # the stored reference in array[1], if present, or seeks to the position from array[0], reads a line, json_decodes it,
17 # stores that reference into array[1] and returns that. That reference will be used from that time onwards.
19 # The routine writeoutjson() will (very) lazily write out a copy of %u WITHOUT STORING ANY EXTRA CURRENTLY UNREFERENCED CALLSIGN
20 # records to users.v4.n. It, in effect, does a sort of random accessed merge of the current user file and any "in memory"
21 # versions of any user record. This can be done with a spawned command because it will just be reading %u and merging
22 # loaded records, not altering the current users.v4 file in any way.
24 # %u -> $u{call} -> [position of json line in users.v4 (, reference -> {call=>'G1TLH', ...} if this record is in use)].
26 # On my machine, it takes about 250mS to read the entire users.v4 file of 190,000 records and to create a
27 # $u{callsign}->[record position in users.v4] for every callsign in the users.v4 file. Loading ~19,000 records
28 # (read from disk, decode json, store reference) takes about 110mS (or 580nS/record).
30 # A periodic dump of users.v4.n, with said ~19,000 records in memory takes about 750mS to write (this can be speeded up,
31 # by at least a half, if it becomes a problem!). As this periodic dump will be spawned off, it will not interrupt the data
34 # This is the first rewrite of DXUsers since inception. In the mojo branch we will no longer use Storable but use JSON instead.
35 # We will now be storing all the keys in memory and will use opportunistic loading of actual records in "get()". So out of
36 # say 200,000 known users it is unlikely that we will have more than 10% (more likely less) of the user records in memory.
37 # This will mean that there will be a increase in memory requirement, but it is modest. I estimate it's unlikely be more
40 # At the moment that means that the working users.v4 is "immutable".
42 # In normal operation, when first calling 'init()', the keys and positions will be read from the newer of users.v4.n and
43 # users.v4. If there is no users.v4.n, then users.v4 will be used. As time wears on, %u will then accrete active user records.
44 # Once an hour the current %u will be saved to users.v4.n.
46 # If it becomes too much of a problem then we are likely to chuck off "close()d" users onto the end of the current users.v4
47 # leaving existing users intact, but updating the pointer to the (now cleared out) user ref to the new location. This will
48 # be a sort of write behind log file. The users.v4 file is still immutable for the starting positions, but any chucked off
49 # records (or even "updates") will be written to the end of that file. If this has to be reread at any time, then the last
50 # entry for any callsign "wins". But this will only happen if I think the memory requirements over time become too much.
52 # As there is no functional difference between the users.v4 and export_user generated "user_json" file(s), other than the latter
53 # will be in sorted order with the record elements in "canonical" order. There will now longer be any code to execute to
54 # "restore the users file". Simply copy one of the "user_json" files to users.v4, remove users.v4.n and restart.
56 # Hopefully though, this will put to rest the need to do all that messing about ever again... Pigs may well be seen flying over
57 # your node as well :-)
72 use Data::Structure::Util qw(unbless);
73 use Time::HiRes qw(gettimeofday tv_interval);
78 use vars qw(%u $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $v3 $v4);
82 $lastoperinterval = 60*24*60*60;
85 $tooold = 86400 * 365 + 31; # this marks an old user who hasn't given enough info to be useful
90 our $maxconnlist = 3; # remember this many connection time (duration) [start, end] pairs
92 our $newusers = 0; # per execution stats
98 my $ifh; # the input file, initialised by readinjson()
101 # hash of valid elements and a simple prompt
103 call => '0,Callsign',
104 alias => '0,Real Callsign',
107 lat => '0,Latitude,slat',
108 long => '0,Longitude,slong',
110 email => '0,E-mail Address,parray',
111 priv => '9,Privilege Level',
112 lastin => '0,Last Time in,cldatetime',
113 passwd => '9,Password,yesno',
114 passphrase => '9,Pass Phrase,yesno',
115 addr => '0,Full Address',
116 'sort' => '0,Type of User', # A - ak1a, U - User, S - spider cluster, B - BBS
117 xpert => '0,Expert Status,yesno',
119 node => '0,Last Node',
120 homenode => '0,Home Node',
121 lockout => '9,Locked out?,yesno', # won't let them in at all
122 dxok => '9,Accept DX Spots?,yesno', # accept his dx spots?
123 annok => '9,Accept Announces?,yesno', # accept his announces?
124 lang => '0,Language',
125 hmsgno => '0,Highest Msgno',
126 group => '0,Group,parray', # used to create a group of users/nodes for some purpose or other
127 buddies => '0,Buddies,parray',
128 isolate => '9,Isolate network,yesno',
129 wantbeep => '0,Req Beep,yesno',
130 wantann => '0,Req Announce,yesno',
131 wantwwv => '0,Req WWV,yesno',
132 wantwcy => '0,Req WCY,yesno',
133 wantecho => '0,Req Echo,yesno',
134 wanttalk => '0,Req Talk,yesno',
135 wantwx => '0,Req WX,yesno',
136 wantdx => '0,Req DX Spots,yesno',
137 wantemail => '0,Req Msgs as Email,yesno',
138 pagelth => '0,Current Pagelth',
139 pingint => '9,Node Ping interval',
140 nopings => '9,Ping Obs Count',
141 wantlogininfo => '0,Login Info Req,yesno',
142 wantgrid => '0,Show DX Grid,yesno',
143 wantann_talk => '0,Talklike Anns,yesno',
144 wantpc16 => '9,Want Users from node,yesno',
145 wantsendpc16 => '9,Send PC16,yesno',
146 wantroutepc19 => '9,Route PC19,yesno',
147 wantusstate => '0,Show US State,yesno',
148 wantdxcq => '0,Show CQ Zone,yesno',
149 wantdxitu => '0,Show ITU Zone,yesno',
150 wantgtk => '0,Want GTK interface,yesno',
151 wantpc9x => '0,Want PC9X interface,yesno',
152 lastoper => '9,Last for/oper,cldatetime',
153 nothere => '0,Not Here Text',
154 registered => '9,Registered?,yesno',
155 prompt => '0,Required Prompt',
156 version => '1,Version',
158 believe => '1,Believable nodes,parray',
159 lastping => '1,Last Ping at,ptimelist',
160 maxconnect => '1,Max Connections',
161 startt => '0,Start Time,cldatetime',
162 connlist => '1,Connections,parraydifft',
169 my $name = $AUTOLOAD;
171 return if $name =~ /::DESTROY$/;
174 confess "Non-existant field '$AUTOLOAD'" if !$valid{$name};
175 # this clever line of code creates a subroutine which takes over from autoload
176 # from OO Perl - Conway
177 *$AUTOLOAD = sub {@_ > 1 ? $_[0]->{$name} = $_[1] : $_[0]->{$name}};
184 # initialise the system
190 my $convert = "$main::root/perl/convert-users-v3-to-v4.pl";
193 $json = JSON->new()->canonical(1);
194 $filename = localdata("users.v4");
196 if (-e $filename || -e "$filename.n" || -e "$filename.o") {
199 # if (-e localdata('users.v3')) {
200 # LogDbg('DXUser', "Converting " . localdata('users.v3') . " to new json version of users file, please wait");
207 die "User file $filename missing, please run $convert or copy a user_json backup from somewhere\n" unless -e "$filename.n" || -s $filename;
210 copy $filename, "$filename.n" unless -e "$filename.n";
216 # with extreme prejudice
217 unlink "$main::data/users.v4";
218 unlink "$main::local_data/users.v4";
222 # periodic processing
226 # if ($main::systime > $lasttime + 15) {
227 # #$dbm->sync if $dbm;
228 # $lasttime = $main::systime;
243 # new - create a new user
250 my $self = bless {call => $call, 'sort'=>'U'}, $pkg;
258 # $call =~ s/-\d+$//o;
260 confess "can't create existing call $call in User\n!" if $u{$call};
262 my $self = $pkg->alloc($call);
263 $u{$call} = [0, $self];
271 # get - get an existing user - this seems to return a different reference everytime it is
278 my $nodecode = shift;
280 return undef unless $ref;
283 $ifh->seek($ref->[0], 0);
284 my $l = $ifh->getline;
286 my ($k,$s) = split /\t/, $l;
287 return $s if $nodecode;
288 my $j = json_decode($s);
294 } elsif ($nodecode) {
295 return json_encode($ref->[1]);
301 # get an "ephemeral" reference - i.e. this will give you new temporary copy of
302 # the call's user record, but without storing it (if it isn't already there)
304 # This is not as quick as get()! But it will allow safe querying of the
305 # user file. Probably in conjunction with get_some_calls feeding it.
307 # NOTE: for cached records this, in effect, is a faster version of Storable's
308 # dclone - only about 3.5 times as fast!
317 return json_decode(json_encode($ref->[1]));
319 $ifh->seek($ref->[0], 0);
320 my $l = $ifh->getline;
322 my ($k,$s) = split /\t/, $l;
323 my $j = json_decode($s);
332 # get an existing record either from the channel (if there is one) or from the database
334 # It is important to note that if you have done a get (for the channel say) and you
335 # want access or modify that you must use this call (and you must NOT use get's all
336 # over the place willy nilly!)
338 # NOTE: mojo branch with newusers system:
339 # There is no longer any function difference between get_current()
340 # and get() as they will always reference the same record as held in %u. This is because
341 # there is no more (repeated) thawing of stored records from the underlying "database".
343 # BUT: notice the difference between this and the get_tmp() function. A get() will online an
344 # othewise unused record, so for poking around looking for that locked out user:
345 # MAKE SURE you use get_tmp(). It will likely still be quicker than DB_File and Storable!
352 # my $call = uc shift;
354 # my $dxchan = DXChannel::get($call);
356 # my $ref = $dxchan->user;
357 # return $ref if $ref && UNIVERSAL::isa($ref, 'DXUser');
359 # dbg("DXUser::get_current: got invalid user ref for $call from dxchan $dxchan->{call} ". ref $ref. " ignoring");
365 # get all callsigns in the database
370 return (sort keys %u);
374 # get some calls - provide a qr// style selector string as a partial key
379 my $pattern = shift || qr/.*/;
380 return sort grep {$pattern} keys %u;
384 # if I understand the term correctly, this is a sort of monad.
386 # Scan through the whole user file and select records that you want
387 # to process further. This routine returns lines of json, yu
389 # the CODE ref should look like:
393 # # maybe do a rough check to see if this is a likely candidate
394 # return unless $line =~ /something/;
395 # my $r = json_decode($l);
396 # return (condition ? wanted thing : ());
405 if (ref($c) eq 'CODE') {
406 foreach my $k (get_all_calls()) {
407 my $l = get($k, 1); # get the offline json line or the jsoned online version
408 push @out, $c->($k, $l) if $l;
411 dbg("DXUser::scan supplied argument is not a code ref");
423 confess "Trying to put nothing!" unless $self && ref $self;
424 $self->{lastin} = $main::systime;
425 ++$modusers; # new or existing, it's still been modified
444 eval { $ref = $json->decode($s) };
446 return bless $ref, 'DXUser';
448 LogDbg('DXUser', "DXUser::json_decode: on '$s' $@");
457 my $s = $json->encode($ref);
458 bless $ref, 'DXUser';
463 # del - delete a user
469 my $call = $self->{call};
472 --$cachedusers if $u{$call}->[1];
477 # close - close down a user
485 $self->{lastin} = $main::systime;
486 # add a record to the connect list
487 my $ref = [$startt || $self->{startt}, $main::systime];
488 push @$ref, $ip if $ip;
489 push @{$self->{connlist}}, $ref;
490 shift @{$self->{connlist}} if @{$self->{connlist}} > $maxconnlist;
503 # return a list of valid elements
513 # export the database to an ascii file
520 my $fn = $name || localdata("user_json"); # force use of local_data
521 my $ta = [gettimeofday];
524 move "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo";
525 move "$fn.ooo", "$fn.oooo" if -e "$fn.ooo";
526 move "$fn.oo", "$fn.ooo" if -e "$fn.oo";
527 move "$fn.o", "$fn.oo" if -e "$fn.o";
528 move "$fn", "$fn.o" if -e "$fn";
530 my $json = JSON->new;
531 $json->canonical(1);;
536 my $fh = new IO::File ">$fn" or return "cannot open $fn ($!)";
540 foreach my $k (sort keys %u) {
542 if ($r->{sort} eq 'U' && !$r->{priv} && $main::systime > $r->{lastin}+$tooold ) {
543 unless ($r->{lat} || $r->{long} || $r->{qra} || $r->{qth} || $r->{name}) {
544 LogDbg('export', "DXUser::export deleting $k - too old, last in " . cldatetime($r->lastin) . " " . difft([$r->lastin, $main::systime]));
550 eval {$val = json_encode($r);};
552 LogDbg('export', "DXUser::export error encoding call: $k $@");
556 $fh->print("$k\t$val\n");
561 my $t = _diffms($ta);
562 my $s = qq{Exported users to $fn - $count Users $del Deleted $err Errors in $t mS ('sh/log Export' for details)};
563 LogDbg('DXUser', $s);
571 # add one or more groups
575 my $ref = $self->{group} || [ 'local' ];
576 $self->{group} = $ref if !$self->{group};
577 push @$ref, @_ if @_;
580 # remove one or more groups
584 my $ref = $self->{group} || [ 'local' ];
587 $self->{group} = $ref if !$self->{group};
589 @$ref = map { my $a = $_; return (grep { $_ eq $a } @in) ? () : $a } @$ref;
592 # does this thing contain all the groups listed?
596 my $ref = $self->{group};
599 return 0 if !$ref || @_ == 0;
600 return 1 if @$ref == 0 && @_ == 0;
601 for ($n = 0; $n < @_; ) {
604 $n++ if grep $_ eq $a, @_;
610 # simplified group test just for one group
615 my $ref = $self->{group};
618 return grep $_ eq $s, $ref;
621 # set up a default group (only happens for them's that connect direct)
625 $self->{group} = [ 'local' ];
628 # set up empty buddies (only happens for them's that connect direct)
632 $self->{buddies} = [ ];
636 # return a prompt for a field
641 my ($self, $ele) = @_;
645 # some variable accessors
649 @_ ? $self->{'sort'} = shift : $self->{'sort'} ;
654 # want is default = 1
661 $self->{$s} = $val if defined $val;
662 return exists $self->{$s} ? $self->{$s} : 1;
665 # wantnot is default = 0
672 $self->{$s} = $val if defined $val;
673 return exists $self->{$s} ? $self->{$s} : 0;
678 return _want('beep', @_);
683 return _want('ann', @_);
688 return _want('wwv', @_);
693 return _want('wcy', @_);
698 return _want('echo', @_);
703 return _want('wx', @_);
708 return _want('dx', @_);
713 return _want('talk', @_);
718 return _want('grid', @_);
723 return _want('email', @_);
728 return _want('ann_talk', @_);
733 return _want('pc16', @_);
738 return _want('sendpc16', @_);
743 return _want('routepc16', @_);
748 return _want('usstate', @_);
753 return _want('dxcq', @_);
758 return _want('dxitu', @_);
763 return _want('gtk', @_);
768 return _want('pc9x', @_);
775 $self->{wantlogininfo} = $val if defined $val;
776 return $self->{wantlogininfo};
782 return $self->{sort} =~ /^[ACRSX]$/;
788 return grep $_ eq 'local_node', @{$self->{group}};
794 return $self->{sort} =~ /^[UW]$/;
800 return $self->{sort} eq 'W';
806 return $self->{sort} eq 'B';
812 return $self->{sort} eq 'S';
818 return $self->{sort} eq 'C';
824 return $self->{sort} eq 'X';
830 return $self->{sort} eq 'R';
836 return $self->{sort} eq 'A';
842 delete $self->{passwd};
849 delete $self->{passphrase};
857 $self->{believe} ||= [];
858 unless (grep $_ eq $call, @{$self->{believe}}) {
859 push @{$self->{believe}}, $call;
868 if (exists $self->{believe}) {
869 $self->{believe} = [grep {$_ ne $call} @{$self->{believe}}];
870 delete $self->{believe} unless @{$self->{believe}};
878 return exists $self->{believe} ? @{$self->{believe}} : ();
885 $self->{lastping} ||= {};
886 $self->{lastping} = {} unless ref $self->{lastping};
887 my $b = $self->{lastping};
888 $b->{$call} = shift if @_;
893 # read in the latest version of the user file. As this file is immutable, the file one really wants is
894 # a later (generated) copy. But, if the plain users.v4 file is all we have, we'll use that.
903 my $ta = [gettimeofday];
908 if (-e $nfn && -e $fn && (stat($nfn))[9] > (stat($fn))[9]) {
909 # move the old file to .o
915 # if we don't have a users.v4 at this point, look for a backup users.v4.json, users.v4.n then users.v4.o
917 move($nfn, $fn) unless -e $fn; # the users.v4 isn't there (maybe convert-users-v3-to-v4.pl
918 move("$fn.json", $fn); # from a run of convert-users-v3-to-v4.pl
919 move($ofn, $fn) unless -e $fn; # desperate now...
925 LogDbg("DXUser","DXUser::readinjson: opening $fn as users file");
926 $ifh = IO::File->new("+<$fn") or die "Cannot open $fn ($!)";
928 my $pos = $ifh->tell;
938 # $ifh is "global" and should not be closed
940 LogDbg('DXUser',"DXUser::readinjson $count record headers read from $fn in ". _diffms($ta) . " mS");
941 return $totusers = $count;
945 # Write a newer copy of the users.v4 file to users.v4.n, which is what will be read in.
946 # This means that the existing users.v4 is not touched during a run of dxspider, or at least
951 my $ofn = shift || "$filename.n";
952 my $ta = [gettimeofday];
954 my $ofh = IO::File->new(">$ofn") or die "$ofn write error $!";
957 for my $k (sort keys %u) {
961 print $ofh "$k\t$l\n";
964 LogDbg('DXUser', "DXUser::writeoutjson callsign $k not found")
969 LogDbg('DXUser',"DXUser::writeoutjson $count records written to $ofn in ". _diffms($ta) . " mS");