#!/usr/bin/perl -w
#
# unicode2names: Find all the Unicode characters and turn them into entities.
#
# 2006-12-18: Written by Steven J. DeRose.

use strict;

our $VERSION = "2012-09-12";

my $quiet = 0;
my $ordLimit = 127;
my $entDir = "/pmc/load/converter3/dtd/nlm/books/ncbi-book/entities-pi";
my @entityNameList = ();
my $entFormat = "%05x";

# Process options
while ($ARGV[0]) {
	if (index($ARGV[0],"--")==0) { $ARGV[0] = substr($ARGV[0],1); }
	if ($ARGV[0] eq "-q") {
	   $quiet = 1;
	}
	elsif ($ARGV[0] eq "-ascii") {
		$ordLimit = 127;
	}
	elsif ($ARGV[0] eq "-latin1") {
		$ordLimit = 255;
	}
	elsif ($ARGV[0] eq "-entformat") {
		die "-entformat option not yet supported.\n";
		shift;
	}
	elsif ($ARGV[0] eq "-version") {
		warn "Version of $VERSION, by Steven J. DeRose.\n";
		exit;
	}
	elsif (substr($ARGV[0],0,1) eq "-") {
	   ($ARGV[0] eq '-h' or $ARGV[0] eq '-help')
	      || print "Unknown option '$ARGV[0]'.\n";
	   system "perldoc $0";
	   exit;
	}
    else {
	   last;
    }
    shift;
} # options

my $file = $ARGV[0];
(-e $file) || die "Couldn't find file '$file'.\n";


###############################################################################

# Set up mapping from Unicode code points to XML entity names
makeEntityNameList();

if ($verbose) {
	print "\nTable of entity names:\n";
	for (my $i=0; $i<scalar @entityNameList; $i++) {
		print "$i: '$entityNameList[$i]'\n";
	}
}

# Copy stdin, making the changes:
my $errCount = 0;
while (my $rec = <>) {
	while ($rec =~ m/&\#[xX]?[0-9a-fA-F];/) {
		# Handle numeric entity references here
	}
	for (my $i=0; $i<length($rec); $i++) {
		$c = substr($rec,$i,1);
		$cn = ord($c);
		$chex = sprintf($entFormat,$cn);
		if ($cn < $ordLimit) {
			print $c;
		}
		elsif ($entNameList[$cn] ne "") {
			print "&$entNameList[$cn];";
		}
		else {
			$errCount++;
			print "<!-- couldn't name entity -->&#x$chex;";
		}
	}
}

if ($errCount > 0) {
	warn "\nTotal errors: $errCount.\n";
}

exit;


###############################################################################


# Extract a list of entity names and numeric values, to make an array where
# we'll look up names by value.
sub makeEntityNameList {
	($quiet) || warn "Looking for entity declarations in '$entDir'.\n";
	(-d $entDir) || die "Can't find entity declarations at $entDir.\n";
	system "ls $entDir";
	my $loadCount = 0;
	my $errCount = 0;
	my $dupCount = 0;
	$cmd = "ls $entDir\/\*\.ent";
	#warn "running: '$cmd'.\n";
	my @entFiles = `$cmd`;
	if (scalar @entFiles <=0) {
		die "No .ent files found in $entDir.\n";
	}
	foreach my $f (@entFiles) {
		chomp $f;
		open IN, "<$f";
		$inComment = 0;
		while ($entdcl = <IN>) {
			chomp $entdcl;
			# Handle full and partial comments
			$entDcl =~ s/<!--.*-->//;
			if ($entDcl =~ s/<!--.*$//) {
				$inComment = 1;
				# still process any prior text
			}
			elsif ($inComment) {
				if ($entDcl =~ s/^.*-->//) { $inComment = 0; }
				else { next; }
			}
				
			($entName = $entDcl) =~ s/(\s*<!ENTITY\s+)//;
			if ($1 eq "") { next; }
			$entName =~ s/\s+.*$//;
			($verbose) && warn "Dcl for '$entName': \t'$entDcl'\n";
			if ($entName =~ m/[^-_.a-zA-Z0-9]/) {
				warn "Invalid entity name '$entName' in file '$f'.\n";
				$errCount++;
				next;
			}
			($ehex = $entdcl) =~ s/.*\s+x='//;
			$ehex =! s/'.*//;
			$entValue = hex($ehex);
			if ($entNameList[$entValue]) {
				warn "Duplicate declaration for position 0x$ehex (0d$entValue): "
					. "'$entNameList[$entValue]' vs. '$entName'.\n";
				$dupCount++;
			}
			else {
				$entNameList[$entValue] = $entName;
				$loadCount++;
			}
		} # while not EOF
		close IN;
	} # foreach .ent file
	($quiet) || warn "\nEntity declaractions: $loadCount loaded, $errCount"
		. " errors, and $dupCount duplicates, from "
		. scalar @entFiles . " files.\n";
} # makeEntityNameList


###############################################################################

=pod

	print "

=head1 Usage
   unicode2names [options]
   Convert unicode characters and numeric references in stdin, into
   XML named character entity references. Reads the table of names from
   '$entDir'.
   ******* Numeric entity reference handling is not yet implemented *******
           Entity declarations are missed if not all on one line

=head1 Options

=over


   -ascii      Convert chars over 127 to entities.
   -latin1     (default) convert chars over 255 to entities.
   -entformat  Change format for unmapped entities (default = %05x)
   -q          Suppress most messages.
   -version    Display version info and exit.
";
}