#!/usr/bin/perl

# A program to clean up the News System Spool.  The program has two 
# functions:
#
#     1.  Create "rmdir" commands to remove directories from the News
#         System Spool that do not belong to active NewsGroups.
#
#     2.  Create "rm" commands to remove files from the News System
#         Spool for all articles that are not contained in the News
#         History file.
#
#     3.  Create "rm" commands to remove bad symbolic links from the
#         News System Spool.  These links are zero length and point to
#         the current directory.
#
# Operational notes:
#
#     1.  Run the program monthly from cron to clean up dead files,
#         bad links, and unused directory entries.
#
#     2.  It is best to clean up the spool after/before news.daily
#         runs.  If they run concurrently, problems can arise because
#         news.daily updates the history file and then deletes files
#         while newspool-clean reads the News Spool and then reads the 
#         history file.  If news.daily and newspool-clean must run
#         concurrently, make sure that news.daily starts first with
#         the "delayrm" option and that newspool-clean doesn't start
#         until the new history file is completed.  Beware, that 
#         newspool-clean may generate "Invalid entry" error messages
#         for incomplete symbolic links and many of the rm-cmds will
#         give error messages for files that news.daily deleted.
#
#     3.  Until you have confidence in the program, use the "-f"
#         option to direct the output commands into a file.  Later
#         you can pipe the output directly to sh.
#
#     4.  Depending on the News Spool size, it may take 12 hours to
#         scan through all the directories and files.  Reading the 
#         history file may take up to an hour.  These times are
#         dependent on how busy are the disks holding the News Spool.
#
# This program is based on code I adopted from:
#
#         "findmissing" by Harry Bochner and David Barr which removes
#         articles not found in the history file.
#
#         a program by Erland Sommarskog and John Milburn which removes
#         directories for newsgroups not found in the active file.
#
#         a recursive subroutine for reading directories that I found
#         in the Camel book, Programming Perl, by Larry Wall and Randal
#         Schwartz.
#
# Please let me know of any problems or suggestions for improvement.
#
# The latest version can be found at ftp://ftp.bihs.net/pub/inn
#
# Butch Kemper <kemper@bihs.net>

# History:
# 
#  v1.0b1   -  6 Jun 97
#  v1.0b2   - 20 Jun 97 - Added command line options.
#  v1.0b3   -  2 Jul 97 - Fixed loop caused by file with incomplete symbolic
#                         link.
#                         Sort directory entries before processing.
#                         Output message for large directories.
#  v1.0b4   - 12 Jul 97 - Subtle changes to speed up processing.
#                         Add option for large directory threshold.
#  v1.0b5   - 20 Jul 97 - Output "rm" commands in normal sorted order and
#                         "rmdir" commands in reverse sorted order.
#  v1.0b6   -  4 Aug 97 - Warn about files with names having non-digits
#                         and delete them.
#  v1.0b7   - 17 Aug 97 - Add requirement for Perl version 5.003
#             31 Aug 97 - Add documentation about where to find the newest
#                         version.
#

$VERSION         = "v1.0b7";    # version Number

require 5.003;                  # required Perl version

#
# Change these variables to match your installation or modify with command
# line entries
#

$SPOOL   = "/newsbin";          # the root directory of the news spool
$ACTIVE  = "/news/active";      # the news system active file
$HISTORY = "/news/history";     # the news system history file
$MAXDIR  = 40;                  # directory size threshold

#
# Special files/directories that are NOT processed are listed in the next
# variable.
#

%EXCLUDES      = ('.',1,
                  '..',1,
                  'in.coming',1,
                  'out.going',1,
                  'news.archive',1,
                  'over.view',1,
                  'lost+found',1,
                  'log',1,
                  'OLD',1);

#
# Initialize other variables
#

$VERBOSE         = 0;                # debugging information level
$OUTPUT          = ">-";             # the default output file
$OFFSET          = "";               # the top level newsgroup

$rmdir_count     = 0;                # count of directories removed
$rm_count        = 0;                # count of files removed
$active_count    = 0;                # count of newgroups in active file
$article_count   = 0;                # count of articles in history file
$depth           = 0;                # depth of newsgroup directories
$bad_links       = 0;                # count of bad symbolic links removed

($PROGNAME = $0) =~ s/.*\///;        # get program name

#
# Process command line options
#

use Getopt::Std;                     # define option processing routines

getopts ('a:df:h:m:o:s:') ||        # scan the command line options
    die ("Usage $PROGNAME [-a file] [-d] [-f file] [-h file] [-m num]" .
         " [-o group] [-s dir]\n"                                .
         "      -a  file  = complete path to active file\n"           .
         "      -d          output additional status messages\n"      .
         "      -f  file  = complete path to output file\n"           .
         "      -h  file  = complete path to history file\n"          .
         "      -m  num   = directory size warning threshold\n"       .
         "      -o  group = offset into newsgroups hierarchy\n"       .
         "      -s  dir   = complete path to top newgroup directory\n");

$VERBOSE = 1        if (defined $opt_d); # turn on debugging messages
$ACTIVE = "$opt_a"  if (defined $opt_a); # alternate active file
$HISTORY= "$opt_h"  if (defined $opt_h); # alternate history file
$OUTPUT = ">$opt_f" if (defined $opt_f); # alternate output file
$SPOOL  = "$opt_s"  if (defined $opt_s); # alternate spool directory
$OFFSET = "$opt_o"  if (defined $opt_o); # alternate newsgroup offset
$SPOOLX = "$SPOOL";                      # adjust spool
$SPOOLX = "$SPOOL/$OFFSET"               #  directory start
          if (! $OFFSET eq "");          #   if offset set
$MAXDIR = $opt_m    if (defined $opt_m); # directory size warning threshold
if (defined $opt_m) {                    # if option "m" is defined
   if ($opt_m =~ /^\d+$/) {              # must only be digits
      $MAXDIR = $opt_m;                  # directory size warning threshold
   }
   else {                                # not all digits
      die ("Invalid value for option -m: $opt_m\n");
   }
}

#
# Tell what is happening
#

warn ("Version                  = $VERSION\n"    .
      "Active file              = $ACTIVE\n"     .
      "History file             = $HISTORY\n"    .
      "Output file              = $OUTPUT\n"     .
      "News Spool               = $SPOOL\n"      .
      "NewsGroup Offset         = $OFFSET\n"     .
      "Directory size threshold = $MAXDIR\n"
                                            ) if $VERBOSE;


#
# Read the active file creating an array keyed by the elements of
# each newsgroup name.  For example:
#
#                 comp
#                 comp/sys
#                 comp/sys/mac
#                 comp/sys/mac/comm
#
# Remember the high and low article numbers and allocate a bit string
# of zeros to represent all possible article numbers in range.

open (ACT, $ACTIVE) ||               # open the active newgroup file
      die "Unable to open active file $ACTIVE: $!\n";
warn ("Scan of News System Active File started\n") if $VERBOSE;

while (<ACT>) {                         # read record for each active newgroup
  ($group,$hi,$lo,$flags) = split;      # split on whitespace
  next if !($hi =~ /^\d+$/o);           # must be 
  next if !($lo =~ /^\d+$/o);           #  only digits
  @parts = split (/\./o, $group);       # split on "."
  foreach $i ($[..$#parts) {            # step through pieces
     $key = join ("/", @parts[$[..$i]); # put "/" between parts
     $active{$key} = 1;                 # define entry
  }
  $active_count++;                      # count NewsGroups
}
close (ACT);                            # close the active newgroup file

die ("Invalid active file $ACTIVE: $active_count records read\n")
    if ($active_count <= 0);

warn ("Scan of News System Active File completed: $active_count records\n")
     if $VERBOSE;

#
# Open the output file
#

open (OUT, $OUTPUT) ||                  # open output file
  die ("Unable to open output file $OUTPUT: $!\n");
select((select(OUT), $| = 1)[0])        # unbuffer output
  if $VERBOSE;                          #  file

# 
# Open the News System Spool and start processing
#

warn ("Scan of News System Spool started\n")
  if $VERBOSE;

chdir ($SPOOLX) ||                      # top directory of News System Spool
  die ("Unable to change directory to $SPOOLX: $!\n");

&walkdir ($SPOOLX, $OFFSET, $depth);    # scan News System Spool

warn ("Scan of News System Spool completed\n")
  if $VERBOSE;

#
# Scan the News System History File
#

warn ("Scan of News System History started\n")
   if $VERBOSE;
open (HIS, $HISTORY) ||                 # open article history file
  die ("Unable to open history file $HISTORY: $!\n");
while (<HIS>) {                         # read through history file
   chop;                                # remove terminating \n
   warn ("Processed $. history records and $article_count article pointers\n") 
      if ($VERBOSE && $. % 50000 == 0);
   @arts = split;                       # split on whitespace
   next if ($#arts < 2);                # skip entries without articles
   shift @arts; shift @arts;            # skip id and date
   foreach (@arts) {                    # loop through all articles
      ($group, $num) = split /\//o;     # split on slash
      next unless defined ($num);       # must be an
      next unless $num =~ /^\d+$/o;     #  article number
      $group =~ s/\./\//go;             # replace "." with "/"
      $article_count++;                 # count the article
      next if (!defined ($articles_lo{$group})); # newsgroup not found
      $i = $articles_lo{$group};        # lo article number
                                        # validate article number
      if ($num >= $i && $num <= $articles_hi{$group}) {

         # Zero the bit that corresponds to the article number
         vec ($articles_bits{$group}, $num - $i, 1) = 0;
      }
   }
}
close (HIS);                            # close history file

# Each "1" bit now represents an article not contained in the History file
foreach $group (sort keys %articles_bits) {     # process each bit string
   next if (! defined ($articles_lo{$group}));  # skip if no articles
   $string = unpack ("b*", $articles_bits{$group});  # extract bit string
   next unless $string =~ /1/o;         # skip if no "1" bits
   $path = $group;                      # newsgroup path to directory
   $path =~ s/\./\//go;                 # replace "." with "/"
   $i = $articles_lo{$group};           # lowest article number
   for $j (split(/x*/o,$string)) {      # step through the bits
      if ($j == 1) {                    # process each "1" bit
         push (@RMFILES, "$SPOOL/$path/$i"); # file to be deleted
         $rm_count++;                   # count the command
      }
      $i++;                             # next article number
   }
}
warn ("Scan of News System History completed\n")
   if $VERBOSE;

#
# Sort names and then write commands to remove files and directories
#
foreach $name (sort @RMFILES) {            # File name with
   print (OUT "rm    $name\n");            #  complete path
}
foreach $name (sort reverse_sort @RMDIRS) { # Directory name with
   print (OUT "rmdir $name\n");            #  complete path
}

#
# End of Program
#

close (OUT);                               # output file

warn ("Number of NewsGroups = $active_count\n"          .
      "Number of Articles deleted = $rm_count\n"        .
      "Number of Directories deleted = $rmdir_count\n"  .
      "Number of Bad Links deleted = $bad_links\n");

exit (0);
#
# Reverse Order Sort Routine
#

sub reverse_sort {$b cmp $a;}

#
# Recursive subroutine to read through directories
#

sub walkdir {
   my ($path, $group, $depth) = @_;        # input arguments
   my ($name, $depth_new, @files, $string,$group_new, $hi, $lo, @filenames);
   my ($this_dev, $this_ino, $mode, $nlink, $uid, $gid, $rdev);
   my ($size, $atime, $mtime, $ctime, $blksize, $blocks, $dev, $ino);

   opendir (DIR, $path) ||                 # open directory
      die ("Unable to open directory $path: $!\n");
   @filenames = sort (readdir (DIR));      # get all file names from directory
   closedir (DIR);                         # close directory

   ($this_dev, $this_ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,
               $mtime,$ctime,$blksize,$blocks) =
               stat ($path);               # Identify this directory

   warn ("Large directory found: $path with $blocks blocks\n")
      if ($MAXDIR > 0  && $MAXDIR < $blocks);  # watch for large directories

   # process each name
   $hi = $lo = 0;                          # initialize boundaries
   foreach $name (@filenames) {            # process each filename
      next if defined ($EXCLUDES{$name});  # skip special directories
      ($dev,$ino) = stat ($name);          # get info about the filename
                                           # look for bad symbolic links
      if (($this_dev == $dev) && ($this_ino == $ino)) {
         warn ("Zero length directory: $path/$name\n")
            if $VERBOSE;
         push (@RMFILES, "$path/$name");   # file to be deleted
         $bad_links++;                     # count the bad link
         next;                             # process next entry
      }                                    # end of bad symbolic link

      if (-d _) {                          # process a directory

        warn ("Processing directory: $path/$name\n") 
           if ($VERBOSE || $depth == 0);
        $group_new = $name;                # next
        $group_new = "$group/$name"        #  newsgroup
           if ($group ne "");              #   key
        if (!defined ($active{"$group_new"})) { # if not defined then
           push (@RMDIRS, "$path/$name");       # remove the entry
           $rmdir_count++;                      # count the directory
        }
        chdir ("$name") ||                 # next directory down
           die ("Unable to change directory to $path/$name: $!\n");
        $depth_new = $depth + 1;           # count directory levels
        &walkdir ("$path/$name", $group_new, $depth_new);
        chdir ($path) ||                   # back to current directory
           die ("Unable to change directory to $path: $!");
      }                                    # end of directory process

      elsif (-f _) {                       # look for files
         if ($name !~ /^\d+$/o) {          #  with nondigits in name
            warn ("File name with non-digits found: $path/$name\n");
            push (@RMFILES, "$path/$name");# delete file
            $bad_links++;                  # count the bad file
            next;                          # skip remainder of processing
         }
         push (@files, $name);             # save the filename
         $lo = $name if ($lo == 0 || $lo > $name); # save lowest number
         $hi = $name if ($name > $hi);     # save highest number
      }                                    # end of file process

      else {                               # process any other entry
         warn ("Invalid entry: $path/$name\n")
            if $VERBOSE;
         push (@RMFILES, "$path/$name");   # file to be deleted
         $bad_links++;                     # count the bad link
      }                                    # end of any other entry process

   }                                       # all files processed

   if ($lo) {                              # process if any articles exist
      $articles_lo{$group} = $lo;          # remember low article
      $articles_hi{$group} = $hi;          # remember high article
      $string = "\0" x (($hi-$lo+7) / 8);  # fill the string with zeros
      foreach $name (@files) {             # step through the filenames
         vec ($string,$name - $lo, 1) = 1; # set the bit to "1"
      }
      $articles_bits{$group} = $string;    # save the bit string
   }
}                                          # end of walkdir
