X-Git-Url: http://gb7djk.dxcluster.net/gitweb/gitweb.cgi?a=blobdiff_plain;f=perl%2FAnnTalk.pm;h=ee7ea51570164916e925f08f34eb24c79dae46ee;hb=refs%2Fheads%2Fnewdisc;hp=02bb0a9bedca8393479b4ef40668ff6eb7b6b97b;hpb=375f2f00086f959d332837f9035d8db1911678df;p=spider.git

diff --git a/perl/AnnTalk.pm b/perl/AnnTalk.pm
index 02bb0a9b..ee7ea515 100644
--- a/perl/AnnTalk.pm
+++ b/perl/AnnTalk.pm
@@ -17,8 +17,8 @@ use DXVars;
 
 use vars qw(%dup $duplth $dupage $filterdef);
 
-$duplth = 60;					# the length of text to use in the deduping
-$dupage = 5*3600;			# the length of time to hold ann dups
+$duplth = 30;					# the length of text to use in the deduping
+$dupage = 18*3600;				# the length of time to hold ann dups
 $filterdef = bless ([
 			  # tag, sort, field, priv, special parser 
 			  ['by', 'c', 0],
@@ -47,8 +47,7 @@ sub dup
 	chomp $text;
 	unpad($text);
 	$text =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg;
-	$text = Encode::encode("iso-8859-1", $text) if $main::can_encode && Encode::is_utf8($text, 1);
-	$text = pack("C*", map {$_ & 127} unpack("C*", $text));
+#	$text = Encode::encode("iso-8859-1", $text) if $main::can_encode && Encode::is_utf8($text, 1);
 	$text =~ s/[^\#a-zA-Z0-9]//g;
 	$text = substr($text, 0, $duplth) if length $text > $duplth; 
 	my $dupkey = "A$call|$to|\L$text";
@@ -67,6 +66,7 @@ sub is_talk_candidate
 {
 	my ($from, $text) = @_;
 	my $call;
+
 	($call) = $text =~ /^\s*(?:[Xx]|[Tt][Oo]?:?)\s+([\w-]+)/;
 	($call) = $text =~ /^\s*>\s*([\w-]+)\b/ unless $call;
 	($call) = $text =~ /^\s*([\w-]+):?\b/ unless $call;