3 # Test filter partially based on Ambrose's hideous subst.pl code
4 # The idea is that the .tmpl files are not valid HTML, and as a result
5 # HTML::Parse would be completely confused by these templates.
6 # This is just a simple scanner (not a parser) & should give better results.
8 # This script is meant to be a drop-in replacement of text-extract.pl
10 # FIXME: Strings like "<< Prev" confuses *this* filter
15 use vars qw( $input );
16 use vars qw( $debug_dump_only_p );
18 ###############################################################################
21 use vars qw( $re_directive );
23 # $re_directive must not do any backreferences
24 $re_directive = q{<(?:(?i)(?:!--\s*)?\/?TMPL_(?:VAR|LOOP|INCLUDE|IF|ELSE|UNLESS)\b(?:\s+(?:[a-zA-Z][-a-zA-Z0-9]*=)?(?:'[^']*'|"[^"]*"|[^\s<>]+))\s*(?:--)?)>};
27 # Hideous stuff from subst.pl, slightly modified to use the above hideous stuff
28 # Note: The $re_tag's set $1 (<tag), $2 (>), and $3 (rest of string)
29 use vars qw( $re_comment $re_entity_name $re_end_entity $re_etag );
30 use vars qw( $re_tag_strict $re_tag_compat @re_tag );
33 my $etag = $compat? '>': '<>\/';
34 # See the file "subst.pl.test1" for how the following mess is derived
35 # Unfortunately, inserting $re_directive's has made this even messier
36 q{(<\/?(?:|(?:"(?:} . $re_directive . q{|[^"])*"|'(?:} . $re_directive . q{|[^'])*'|--(?:[^-]|-[^-])*--|(?:} . $re_directive . q{|[^-"'} . $etag . q{]|-[^-]))+))([} . $etag . q{])(.*)};
39 $re_comment = '(?:--(?:[^-]|-[^-])*--)';
40 $re_entity_name = '(?:[^&%#;<>\s]+)'; # NOTE: not really correct SGML
41 $re_end_entity = '(?:;|$|(?=\s))'; # semicolon or before-whitespace
42 $re_etag = q{(?:<\/?(?:"[^"]*"|'[^']*'|[^"'>\/])*[>\/])}; # end-tag
43 @re_tag = ($re_tag_strict, $re_tag_compat) = (re_tag(0), re_tag(1));
46 # End of the hideous stuff
48 sub KIND_TEXT () { 'TEXT' }
49 sub KIND_CDATA () { 'CDATA' }
50 sub KIND_TAG () { 'TAG' }
51 sub KIND_DECL () { 'DECL' }
52 sub KIND_PI () { 'PI' }
53 sub KIND_DIRECTIVE () { 'HTML::Template' }
54 sub KIND_COMMENT () { 'COMMENT' } # empty DECL with exactly one SGML comment
55 sub KIND_UNKNOWN () { 'ERROR' }
57 use vars qw( $readahead $lc_0 $lc $syntaxerror_p );
58 use vars qw( $cdata_mode_p $cdata_close );
60 sub extract_attributes ($;$) {
63 $s = $1 if $s =~ /^<\S+(.*)\S$/s; # should be always true
64 for (my $i = 0; $s =~ /^\s+(?:([a-zA-Z][-a-zA-Z0-9]*)=)?('((?:$re_directive|[^'])*)'|"((?:$re_directive|[^"])*)"|(($re_directive|[^\s<>])+))/os;) {
65 my($key, $val, $val_orig, $rest)
66 = ($1, (defined $3? $3: defined $4? $4: $5), $2, $');
68 $attr{+lc($key)} = [$key, $val, $val_orig, $i];
71 if ($s =~ /\S/s) { # should never happen
72 warn "Warning: Strange attribute syntax"
73 . (defined $lc? " in line $lc": '') . ": $s\n";
79 sub next_token_internal (*) {
83 if (!defined $readahead || !length $readahead) {
84 my $next = scalar <$h>;
85 $eof_p = !defined $next;
91 $lc_0 = $lc; # remember line number of first line
92 if ($eof_p && !length $readahead) { # nothing left to do
94 } elsif ($readahead =~ /^\s+/s) { # whitespace
95 ($kind, $it, $readahead) = (KIND_TEXT, $&, $');
96 } elsif ($readahead =~ /^[^<]+/s) { # non-whitespace normal text
97 ($kind, $it, $readahead) = (KIND_TEXT, $&, $');
98 } else { # tag/declaration/processing instruction
102 if ($readahead =~ /^$cdata_close/) {
103 ($kind, $it, $readahead) = (KIND_TAG, $&, $');
106 ($kind, $it, $readahead) = (KIND_TEXT, $readahead, undef);
109 } elsif ($readahead =~ /^$re_tag_compat/os) {
110 ($kind, $it, $readahead) = (KIND_TAG, "$1$2", $3);
112 } elsif ($readahead =~ /^<!--(?:(?!-->).)*-->/s) {
113 ($kind, $it, $readahead) = (KIND_COMMENT, $&, $');
115 warn "Warning: Syntax error in comment at line $lc_0: $&\n";
119 my $next = scalar <$h>;
120 $eof_p = !defined $next;
125 if ($kind ne KIND_TAG) {
127 } elsif ($it =~ /^<!/) {
129 $kind = KIND_COMMENT if $it =~ /^<!--(?:(?!-->).)*-->/;
130 } elsif ($it =~ /^<\?/) {
133 if ($it =~ /^$re_directive/ios && !$cdata_mode_p) {
134 $kind = KIND_DIRECTIVE;
136 ($kind, $it) = (KIND_UNKNOWN, $readahead)
137 if !$ok_p && $eof_p && !length $readahead;
139 return defined $it? (wantarray? ($kind, $it):
140 [$kind, $it]): undef;
146 if (!$cdata_mode_p) {
147 $it = next_token_internal($h);
148 if (defined $it && $it->[0] eq KIND_TAG) { # FIXME
149 ($cdata_mode_p, $cdata_close) = (1, "</$1\\s*>")
150 if $it->[1] =~ /^<(script|style|textarea)\b/i; #FIXME
151 push @$it, extract_attributes($it->[1], $lc); #FIXME
156 my $next = next_token_internal($h);
157 last if !defined $next;
158 if (defined $next && $next->[1] =~ /$cdata_close/i) { #FIXME
159 ($lc, $readahead) = ($lc_prev, $next->[1] . $readahead); #FIXME
162 last unless $cdata_mode_p;
163 $it .= $next->[1]; #FIXME
165 $it = [KIND_CDATA, $it] if defined $it; #FIXME
166 $cdata_close = undef;
168 return defined $it? (wantarray? @$it: $it): undef;
171 ###############################################################################
173 sub debug_dump (*) { # for testing only
175 print "re_tag_compat is /$re_tag_compat/\n";
177 my $s = next_token $h;
178 last unless defined $s;
179 printf "%s\n", ('-' x 79);
180 my($kind, $t, $attr) = @$s; # FIXME
181 printf "%s:\n", $kind;
182 printf "%4dH%s\n", length($t),
183 join('', map {/[\0-\37]/? $_: "$_\b$_"} split(//, $t));
184 if ($kind eq KIND_TAG && %$attr) {
185 printf "Attributes:\n";
186 for my $a (keys %$attr) {
187 my($key, $val, $val_orig, $order) = @{$attr->{$a}};
188 printf "%s = %dH%s -- %s\n", $a, length $val,
189 join('', map {/[\0-\37]/? $_: "$_\b$_"} split(//, $val)),
196 ###############################################################################
198 sub text_extract (*) {
202 my $s = next_token $h;
203 last unless defined $s;
204 my($kind, $t, $attr) = @$s; # FIXME
205 if ($kind eq KIND_TEXT) {
207 $text{$t} = 1 if $t =~ /\S/s;
208 } elsif ($kind eq KIND_TAG && %$attr) {
209 # value [tag=input], meta
210 my $tag = lc($1) if $t =~ /^<(\S+)/s;
211 for my $a ('alt', 'content', 'title', 'value') {
213 next if $a eq 'content' && $tag ne 'meta';
214 next if $a eq 'value' && ($tag ne 'input'
215 || (ref $attr->{'type'} && $attr->{'type'}->[1] eq 'hidden')); # FIXME
216 my($key, $val, $val_orig, $order) = @{$attr->{$a}}; #FIXME
218 $text{$val} = 1 if $val =~ /\S/s;
223 for my $t (keys %text) {
224 printf "%s\n", $t unless $t =~ /^(?:\s|\ )*$/s;
228 ###############################################################################
231 'f|file=s' => \$input,
232 'debug-dump-only-p' => \$debug_dump_only_p,
235 open(INPUT, "<$input") || die "$0: $input: $!\n";
236 if ($debug_dump_only_p) {
239 text_extract(*INPUT);
242 warn "Warning: This input will not work with Mozilla standards-compliant mode\n"