From 03695ce8115026f3bcc6f140036f1c370ae04270 Mon Sep 17 00:00:00 2001
From: acli <acli>
Date: Sun, 22 Feb 2004 09:04:53 +0000
Subject: [PATCH] Try to relax the criteria for allowing groups of tokens
 without TMPL_VAR to be combined together into one string. This seems to have
 the desired effect (that "<b>foo</b> bar" type strings are now recognized in
 one piece).

However, "<h1>foo</h1>\nexplanation"-type things may now also be (arguably
wrongly) recognized as one piece.
---
 misc/translator/TmplTokenizer.pm | 33 +++++++++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/misc/translator/TmplTokenizer.pm b/misc/translator/TmplTokenizer.pm
index e5177c4765..608d141001 100644
--- a/misc/translator/TmplTokenizer.pm
+++ b/misc/translator/TmplTokenizer.pm
@@ -459,9 +459,7 @@ sub _optimize {
 	    && $structure[$#structure]->type == TmplTokenType::TAG
 	    && $structure[$#structure]->string =~ /^<\//s) {
 	my $has_other_tags_p = 0;
-	    printf STDERR "last token %d is type %s: %s\n", $#structure, $structure[$#structure]->type->to_string, $structure[$#structure]->string;
 	for (my $i = 0; $i < $#structure; $i += 1) {
-	    printf STDERR "token %d is type %s: %s\n", $i, $structure[$i]->type->to_string, $structure[$i]->string;
 	    $has_other_tags_p = 1 if $structure[$i]->type == TmplTokenType::TAG;
 	last if $has_other_tags_p;
 	}
@@ -478,11 +476,38 @@ sub _optimize {
     return @structure;
 }
 
+sub looks_plausibly_like_groupable_text_p (@) {
+    my @structure = @_;
+    # The text would look plausibly groupable if all open tags are also closed.
+    my @tags = ();
+    my $error_p = 0;
+    for (my $i = 0; $i <= $#structure; $i += 1) {
+	if ($structure[$i]->type == TmplTokenType::TAG) {
+	    if ($structure[$i]->string =~ /^<([A-Z0-9]+)/i) {
+		push @tags, lc($1);
+	    } elsif ($structure[$i]->string =~ /^<\/([A-Z0-9]+)/i) {
+		if (@tags && lc($1) eq $tags[$#tags]) {
+		    pop @tags;
+		} else {
+		    $error_p = 1;
+		}
+	    }
+	} elsif ($structure[$i]->type != TmplTokenType::TEXT) {
+	    $error_p = 1;
+	}
+    last if $error_p;
+    }
+    return !$error_p && !@tags;
+}
+
 sub next_token {
     my $this = shift;
     my $h = $this->_handle;
     my $it;
     $this->{_queue} = [] unless defined $this->{_queue};
+
+    # Don't reparse anything in the queue. We can put a parametrized token
+    # there if we need to, however.
     if (@{$this->{_queue}}) {
 	$it = pop @{$this->{_queue}};
     } else {
@@ -532,7 +557,9 @@ sub next_token {
 		$it = TmplToken->new($string, TmplTokenType::TEXT_PARAMETRIZED, $it->line_number, $it->pathname);
 		$it->set_form( $form );
 		$it->set_children( @structure );
-	    } elsif ($nonblank_text_p && $structure[0]->type == TmplTokenType::TEXT && $structure[$#structure]->type == TmplTokenType::TEXT) {
+	    } elsif ($nonblank_text_p
+	    && looks_plausibly_like_groupable_text_p( @structure )
+	    && $structure[$#structure]->type == TmplTokenType::TEXT) {
 		# Combine the strings
 		my $string = join('', map { $_->string } @structure);
 		$it = TmplToken->new($string, TmplTokenType::TEXT, $it->line_number, $it->pathname);;
-- 
2.20.1