fix missing callsigns

[spider.git] / perl / USDB.pm

diff --git a/perl/USDB.pm b/perl/USDB.pm
index 3b62fa5d59c5d995b5371a1e4c4a4e9460533b95..69e1ead083446e4fbf3fa5c43900299c1418aee9 100644 (file)
--- a/perl/USDB.pm
+++ b/perl/USDB.pm
@@ -5,13 +5,15 @@
 #
 # 
 
+package USDB;
+
 use strict;
 
 use DXVars;
 use DB_File;
 use File::Copy;
 use DXDebug;
-use Compress::Zlib;
+#use Compress::Zlib;
 
 use vars qw($VERSION $BRANCH);
 $VERSION = sprintf( "%d.%03d", q$Revision$ =~ /(\d+)\.(\d+)/ );
@@ -19,14 +21,19 @@ $BRANCH = sprintf( "%d.%03d", q$Revision$ =~ /\d+\.\d+\.(\d+)\.(\d+)/  || (0,0))
 $main::build += $VERSION;
 $main::branch += $BRANCH;
 
-use vars qw(%db $present);
+use vars qw(%db $present $dbfn);
 
-my $dbfn = "$main::data/usdb.v1";
+$dbfn = "$main::data/usdb.v1";
 
 sub init
 {
        end();
-       tie %db, 'DB_File', $dbfn and $present = 1;
+       if (tie %db, 'DB_File', $dbfn, O_RDONLY, 0664, $DB_BTREE) {
+               $present = 1;
+               dbg("US Database loaded");
+       } else {
+               dbg("US Database not loaded");
+       }
 }
 
 sub end
@@ -64,31 +71,64 @@ sub getcity
 #
 # Note that this removes and overwrites the existing DB file
 # You will need to init again after doing this
-#
+# 
 
 sub load
 {
+       return "Need a filename" unless @_;
+       
        # create the new output file
        my $a = new DB_File::BTREEINFO;
        $a->{psize} = 4096 * 2;
-       my $s;
-       if ($s = -s $dbfn && $s > 1024 * 1024) {
-               $a->{cachesize} = int(($s / (1024*1024)) / 2) * 1024 * 1024;
+       my $s = 0;
+
+       # guess a cache size
+       for (@_) {
+               my $ts = -s;
+               $s = $ts if $ts > $s;
        }
+       if ($s > 1024 * 1024) {
+               $a->{cachesize} = int($s / (1024*1024)) * 3 * 1024 * 1024;
+       }
+
+#      print "cache size " . $a->{cachesize} . "\n";
+       
        my %dbn;
        if (-e $dbfn ) {
-               syscopy($dbfn, "$dbfn.new") or return "cannot copy $dbfn -> $dbfn.new $!";
+               copy($dbfn, "$dbfn.new") or return "cannot copy $dbfn -> $dbfn.new $!";
        }
        
        tie %dbn, 'DB_File', "$dbfn.new", O_RDWR|O_CREAT, 0664, $a or return "cannot tie $dbfn.new $!";
        
        # now write away all the files
+       my $count = 0;
        for (@_) {
-               my $fn = shift;
-               my $f = gzopen($fn, "r") or return "Cannot open $fn $!";
-               while ($f->gzreadline) {
-                       chomp;
-                       my ($call, $city, $state) = split /\|/;
+               my $ofn = shift;
+
+               # conditionally handle compressed files (don't cha just lurv live code, this is
+               # a rave from the grave and is "in memoriam Flossie" the ICT 1301G I learnt on.
+               # {for pedant computer historians a 1301G is an ICT 1301A that has been 
+               # Galdorised[tm] (for instance had decent IOs and a 24 pre-modify instruction)}
+               my $nfn = $ofn;
+               if ($nfn =~ /.gz$/i) {
+                       my $gz;
+                       eval qq{use Compress::Zlib; \$gz = gzopen(\$ofn, "rb")};
+                       return "Cannot read compressed files $@" if $@;
+                       $nfn =~ s/.gz$//i;
+                       my $of = new IO::File ">$nfn" or return "Cannot write to $nfn $!";
+                       my ($l, $buf);
+                       $of->write($buf, $l) while ($l = $gz->gzread($buf));
+                       $gz->gzclose;
+                       $of->close;
+                       $ofn = $nfn;
+               }
+
+               my $of = new IO::File "$ofn" or return "Cannot read $ofn $!";
+
+               while (<$of>) {
+                       my $l = $_;
+                       $l =~ s/[\r\n]+$//;
+                       my ($call, $city, $state) = split /\|/, $l;
                        
                        # lookup the city 
                        my $s = "$city|$state";
@@ -102,12 +142,15 @@ sub load
                                $dbn{'##'} = "$no";
                        }
                        $dbn{$call} = $ctyn; 
+                       $count++;
                }
-               $f->gzclose;
+               $of->close;
+               unlink $nfn;
        }
        
        untie %dbn;
        rename "$dbfn.new", $dbfn;
+       return "$count records";
 }
 
 1;