perl/gen_usdb_data.pl

   1 #!/usr/bin/perl
   2 #
   3 # Something to create my subset of the US call book data,
   4 # in my flat file form, either from the main data base or
   5 # else the daily updates.
   6 #
   7 # You can get the main database from:
   8 #
   9 #   http://wireless.fcc.gov/uls/data/complete/l_amat.zip
  10 #
  11 # The daily data bases are available as a set of seven from here:-
  12 #
  13 #   http://wireless.fcc.gov/uls/data/daily/l_am_sat.zip
  14 #   http://wireless.fcc.gov/uls/data/daily/l_am_sun.zip
  15 #   http://wireless.fcc.gov/uls/data/daily/l_am_mon.zip
  16 #   http://wireless.fcc.gov/uls/data/daily/l_am_tue.zip
  17 #   http://wireless.fcc.gov/uls/data/daily/l_am_wed.zip
  18 #   http://wireless.fcc.gov/uls/data/daily/l_am_thu.zip
  19 #   http://wireless.fcc.gov/uls/data/daily/l_am_fri.zip
  20 #
  21 # this program expects one or more zip files containing the call book
  22 # data as arguments.
  23 #
  24 # Copyright (c) 2002 Dirk Koopman G1TLH
  25 #
  26 # $Id$
  27 #
  28
  29 use strict;
  30
  31 # make sure that modules are searched in the order local then perl
  32 BEGIN {
  33         # root of directory tree for this system
  34         use vars qw($root);
  35
  36         $root = "/spider";
  37         $root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'};
  38
  39         unshift @INC, "$root/perl";     # this IS the right way round!
  40         unshift @INC, "$root/local";
  41 }
  42
  43 use vars qw($VERSION $BRANCH);
  44 $VERSION = sprintf( "%d.%03d", q$Revision$ =~ /(\d+)\.(\d+)/ );
  45 $BRANCH = sprintf( "%d.%03d", q$Revision$ =~ /\d+\.\d+\.(\d+)\.(\d+)/  || (0,0));
  46 $main::build += $VERSION;
  47 $main::branch += $BRANCH;
  48
  49 use DXVars;
  50 use Archive::Zip qw(:ERROR_CODES);
  51 use Archive::Zip::MemberRead;
  52 use IO::File;
  53 use Compress::Zlib;
  54
  55 my $blksize = 1024 * 1024;
  56
  57 STDOUT->autoflush(1);
  58
  59 my $dbrawfn = "$main::data/usdbraw.gz";
  60
  61 rename "$dbrawfn.oo", "$dbrawfn.ooo";
  62 rename "$dbrawfn.o", "$dbrawfn.oo";
  63 rename "$dbrawfn", "$dbrawfn.o";
  64 my $gzfh = gzopen($dbrawfn, "wb") or die "Cannot open $dbrawfn $!";
  65
  66 my $ctycount;
  67
  68 foreach my $argv (@ARGV) {
  69         my $zip = new Archive::Zip($argv) or die "Cannot open $argv $!\n";
  70         print "Doing $argv\n";
  71         handleEN($zip, $argv);
  72         handleAM($zip, $argv);
  73         handleHS($zip, $argv);
  74 }
  75
  76 $gzfh->gzclose;
  77
  78 exit(0);
  79
  80 sub handleEN
  81 {
  82         my ($zip, $argv) = @_;
  83         my $mname = "EN.dat";
  84         my $ofn = "$main::data/$mname";
  85         print "  Handling EN records, unzipping";
  86         if ($zip->extractMember($mname, $ofn) == AZ_OK) {
  87                 my $fh = new IO::File "$ofn" or die "Cannot open $ofn $!";
  88                 if ($fh) {
  89
  90                         print ", storing";
  91
  92                         my $count;
  93                         my $buf;
  94
  95                         while (my $l = $fh->getline) {
  96                                 $l =~ s/[\r\n]+$//;
  97                                 my ($rt,$usi,$ulsfn,$ebfno,$call,$type,$lid,$name,$first,$middle,$last,$suffix,
  98                                         $phone,$fax,$email,$street,$city,$state,$zip,$pobox,$attl,$sgin,$frn) = split /\|/, $l;
  99
 100 #                               print "ERR: $l\n" unless $call && $city && $state;
 101
 102                                 if ($call && $city && $state) {
 103                                         my $rec = uc join '|', $call,$city,$state if $city && $state;
 104                                         $buf .= "$rec\n";
 105                                         if (length $buf > $blksize) {
 106                                                 $gzfh->gzwrite($buf);
 107                                                 undef $buf;
 108                                         }
 109                                         $count++;
 110                                 }
 111                         }
 112                         $gzfh->gzwrite($buf) if length $buf;
 113                         print ", $count records\n";
 114                         $fh->close;
 115                 }
 116                 unlink $ofn;
 117         } else {
 118                 print "EN missing in $argv\n";
 119                 return;
 120         }
 121 }
 122
 123 sub handleAM
 124 {
 125
 126 }
 127
 128 sub handleHS
 129 {
 130
 131 }