2 # This file is part of Koha
3 # Parts copyright 2003-2004 Paul Poulain
4 # Parts copyright 2003-2004 Jerome Vizcaino
5 # Parts copyright 2004 Ambrose Li
12 tmpl_process3.pl - Alternative version of tmpl_process.pl
13 using gettext-compatible translation files
18 #use warnings; FIXME - Bug 2505
22 use File::Temp qw( :POSIX );
24 use VerboseWarnings qw( :warn :die );
26 ###############################################################################
28 use vars qw( @in_dirs @filenames @match @nomatch $str_file $out_dir $quiet );
29 use vars qw( @excludes $exclude_regex );
30 use vars qw( $recursive_p );
31 use vars qw( $pedantic_p );
33 use vars qw( $type ); # file extension (DOS form without the dot) to match
34 use vars qw( $charset_in $charset_out );
36 ###############################################################################
38 sub find_translation ($) {
42 $key = TmplTokenizer::string_canon($key);
43 $key = TmplTokenizer::charset_convert($key, $charset_in, $charset_out);
44 $key = TmplTokenizer::quote_po($key);
46 if (defined $href->{$key} && !$href->{$key}->fuzzy && length Locale::PO->dequote($href->{$key}->msgstr)){
48 return $1 . Locale::PO->dequote($href->{$key}->msgstr);
51 return Locale::PO->dequote($href->{$key}->msgstr);
59 sub text_replace_tag ($$) {
63 # value [tag=input], meta
64 my $tag = lc($1) if $t =~ /^<(\S+)/s;
66 for my $a ('alt', 'content', 'title', 'value', 'label', 'placeholder') {
68 next if $a eq 'label' && $tag ne 'optgroup';
69 next if $a eq 'content' && $tag ne 'meta';
70 next if $a eq 'value' && ($tag ne 'input' || (ref $attr->{'type'} && $attr->{'type'}->[1] =~ /^(?:checkbox|hidden|radio)$/)); # FIXME
72 my($key, $val, $val_orig, $order) = @{$attr->{$a}}; #FIXME
74 my $s = find_translation($val);
75 if ($attr->{$a}->[1] ne $s) { #FIXME
76 $attr->{$a}->[1] = $s; # FIXME
77 $attr->{$a}->[2] = ($s =~ /"/s)? "'$s'": "\"$s\""; #FIXME
85 . join('', map { if ($_ ne '/'){
86 sprintf(' %s="%s"', $_, $attr->{$_}->[1]);
93 $attr->{$a}->[3] <=> $attr->{$b}->[3] #FIXME
103 sub text_replace (**) {
104 my($h, $output) = @_;
106 my $s = TmplTokenizer::next_token $h;
107 last unless defined $s;
108 my($kind, $t, $attr) = ($s->type, $s->string, $s->attributes);
109 if ($kind eq C4::TmplTokenType::TEXT) {
110 print $output find_translation($t);
111 } elsif ($kind eq C4::TmplTokenType::TEXT_PARAMETRIZED) {
112 my $fmt = find_translation($s->form);
113 print $output TmplTokenizer::parametrize($fmt, 1, $s, sub {
115 my($kind, $t, $attr) = ($_->type, $_->string, $_->attributes);
116 $kind == C4::TmplTokenType::TAG && %$attr?
117 text_replace_tag($t, $attr): $t });
118 } elsif ($kind eq C4::TmplTokenType::TAG && %$attr) {
119 print $output text_replace_tag($t, $attr);
120 } elsif ($s->has_js_data) {
121 for my $t (@{$s->js_data}) {
122 # FIXME for this whole block
124 printf $output "%s%s%s", $t->[2], find_translation $t->[3],
127 print $output $t->[1];
130 } elsif (defined $t) {
131 # Quick fix to bug 4472
132 $t = "<!DOCTYPE stylesheet [" if $t =~ /DOCTYPE stylesheet/ ;
139 my($dir, $type, $action) = @_;
140 my $filenames = join ('|', @filenames); # used to update strings from this file
141 my $match = join ('|', @match); # use only this files
142 my $nomatch = join ('|', @nomatch); # do no use this files
144 if (opendir(DIR, $dir)) {
145 my @dirent = readdir DIR; # because DIR is shared when recursing
147 for my $dirent (@dirent) {
148 my $path = "$dir/$dirent";
149 if ($dirent =~ /^\./ || $dirent eq 'CVS' || $dirent eq 'RCS'
150 || (defined $exclude_regex && $dirent =~ /^(?:$exclude_regex)$/)) {
153 my $basename = fileparse( $path );
155 if ( not @filenames or $basename =~ /($filenames)/i )
156 and ( not @match or $basename =~ /($match)/i ) # files to include
157 and ( not @nomatch or $basename !~ /($nomatch)/i ) # files not to include
158 and (!defined $type || $dirent =~ /\.(?:$type)$/) || $action eq 'install';
159 } elsif (-d $path && $recursive_p) {
160 push @it, listfiles($path, $type, $action);
164 warn_normal "$dir: $!", undef;
169 ###############################################################################
171 sub mkdir_recursive ($) {
173 local($`, $&, $', $1);
174 $dir = $` if $dir ne /^\/+$/ && $dir =~ /\/+$/;
175 my ($prefix, $basename) = ($dir =~ /\/([^\/]+)$/s)? ($`, $1): ('.', $dir);
176 mkdir_recursive($prefix) if $prefix ne '.' && !-d $prefix;
178 print STDERR "Making directory $dir...\n" unless $quiet;
179 # creates with rwxrwxr-x permissions
180 mkdir($dir, 0775) || warn_normal "$dir: $!", undef;
184 ###############################################################################
188 my $h = $exitcode? *STDERR: *STDOUT;
190 Usage: $0 create [OPTION]
191 or: $0 update [OPTION]
192 or: $0 install [OPTION]
194 Create or update PO files from templates, or install translated templates.
196 -i, --input=SOURCE Get or update strings from SOURCE directory(s).
197 On create or update can have multiple values.
198 On install only one value.
199 -o, --outputdir=DIRECTORY Install translation(s) to specified DIRECTORY
200 --pedantic-warnings Issue warnings even for detected problems
201 which are likely to be harmless
202 -r, --recursive SOURCE in the -i option is a directory
203 -f, --filename=FILE FILE is a specific filename or part of it.
204 If given, only these files will be processed.
205 On update only relevant strings will be updated.
206 -m, --match=FILE FILE is a specific filename or part of it.
207 If given, only these files will be processed.
208 -n, --nomatch=FILE FILE is a specific filename or part of it.
209 If given, these files will not be processed.
210 -s, --str-file=FILE Specify FILE as the translation (po) file
211 for input (install) or output (create, update)
212 -x, --exclude=REGEXP Exclude dirs matching the given REGEXP
213 --help Display this help and exit
214 -q, --quiet no output to screen (except for errors)
216 The -o option is ignored for the "create" and "update" actions.
217 Try `perldoc $0` for perhaps more information.
222 ###############################################################################
224 sub usage_error (;$) {
225 for my $msg (split(/\n/, $_[0])) {
226 print STDERR "$msg\n";
228 print STDERR "Try `$0 --help for more information.\n";
232 ###############################################################################
235 'input|i=s' => \@in_dirs,
236 'filename|f=s' => \@filenames,
237 'match|m=s' => \@match,
238 'nomatch|n=s' => \@nomatch,
239 'outputdir|o=s' => \$out_dir,
240 'recursive|r' => \$recursive_p,
241 'str-file|s=s' => \$str_file,
242 'exclude|x=s' => \@excludes,
243 'quiet|q' => \$quiet,
244 'pedantic-warnings|pedantic' => sub { $pedantic_p = 1 },
248 VerboseWarnings::set_application_name $0;
249 VerboseWarnings::set_pedantic_mode $pedantic_p;
251 # keep the buggy Locale::PO quiet if it says stupid things
252 $SIG{__WARN__} = sub {
254 print STDERR $s unless $s =~ /^Strange line in [^:]+: #~/s
257 my $action = shift or usage_error('You must specify an ACTION.');
258 usage_error('You must at least specify input and string list filenames.')
259 if !@in_dirs || !defined $str_file;
261 # Type match defaults to *.tt plus *.inc if not specified
262 $type = "tt|inc|xsl|xml|def" if !defined($type);
264 # Check the inputs for being directories
265 for my $in_dir ( @in_dirs ) {
266 usage_error("$in_dir: Input must be a directory.\n"
267 . "(Symbolic links are not supported at the moment)")
271 # Generates the global exclude regular expression
272 $exclude_regex = '(?:'.join('|', @excludes).')' if @excludes;
275 # Generate the list of input files if a directory is specified
276 # input is a directory, generates list of files to process
278 for my $fn ( @filenames ) {
279 die "You cannot specify input files and directories at the same time.\n"
282 for my $in_dir ( @in_dirs ) {
283 $in_dir =~ s/\/$//; # strips the trailing / if any
284 @in_files = ( @in_files, listfiles($in_dir, $type, $action));
287 # restores the string list from file
288 $href = Locale::PO->load_file_ashash($str_file);
290 # guess the charsets. HTML::Templates defaults to iso-8859-1
292 die "$str_file: PO file is corrupted, or not a PO file\n" unless defined $href->{'""'};
293 $charset_out = TmplTokenizer::charset_canon $2 if $href->{'""'}->msgstr =~ /\bcharset=(["']?)([^;\s"'\\]+)\1/;
294 $charset_in = $charset_out;
295 # for my $msgid (keys %$href) {
296 # if ($msgid =~ /\bcharset=(["']?)([^;\s"'\\]+)\1/) {
297 # my $candidate = TmplTokenizer::charset_canon $2;
298 # die "Conflicting charsets in msgid: $charset_in vs $candidate => $msgid\n"
299 # if defined $charset_in && $charset_in ne $candidate;
300 # $charset_in = $candidate;
304 # BUG6464: check consistency of PO messages
305 # - count number of '%s' in msgid and msgstr
306 for my $msg ( values %$href ) {
307 my $id_count = split(/%s/, $msg->{msgid}) - 1;
308 my $str_count = split(/%s/, $msg->{msgstr}) - 1;
309 next if $id_count == $str_count ||
310 $msg->{msgstr} eq '""' ||
311 grep { /fuzzy/ } @{$msg->{_flags}};
313 "unconsistent %s count: ($id_count/$str_count):\n" .
314 " line: " . $msg->{loaded_line_number} . "\n" .
315 " msgid: " . $msg->{msgid} . "\n" .
316 " msgstr: " . $msg->{msgstr} . "\n", undef;
320 # set our charset in to UTF-8
321 if (!defined $charset_in) {
322 $charset_in = TmplTokenizer::charset_canon 'UTF-8';
323 warn "Warning: Can't determine original templates' charset, defaulting to $charset_in\n";
325 # set our charset out to UTF-8
326 if (!defined $charset_out) {
327 $charset_out = TmplTokenizer::charset_canon 'UTF-8';
328 warn "Warning: Charset Out defaulting to $charset_out\n";
330 my $xgettext = './xgettext.pl'; # actual text extractor script
333 if ($action eq 'create') {
334 # updates the list. As the list is empty, every entry will be added
336 warn "Removing empty file $str_file\n";
337 unlink $str_file || die "$str_file: $!\n";
339 die "$str_file: Output file already exists\n" if -f $str_file;
340 my($tmph1, $tmpfile1) = tmpnam();
341 my($tmph2, $tmpfile2) = tmpnam();
342 close $tmph2; # We just want a name
343 # Generate the temporary file that acts as <MODULE>/POTFILES.in
344 for my $input (@in_files) {
345 print $tmph1 "$input\n";
348 warn "I $charset_in O $charset_out";
349 # Generate the specified po file ($str_file)
350 $st = system ($xgettext, '-s', '-f', $tmpfile1, '-o', $tmpfile2,
351 (defined $charset_in? ('-I', $charset_in): ()),
352 (defined $charset_out? ('-O', $charset_out): ())
354 # Run msgmerge so that the pot file looks like a real pot file
355 # We need to help msgmerge a bit by pre-creating a dummy po file that has
356 # the headers and the "" msgid & msgstr. It will fill in the rest.
358 # Merge the temporary "pot file" with the specified po file ($str_file)
359 # FIXME: msgmerge(1) is a Unix dependency
360 # FIXME: need to check the return value
361 unless (-f $str_file) {
362 local(*INPUT, *OUTPUT);
363 open(INPUT, "<$tmpfile2");
364 open(OUTPUT, ">$str_file");
372 $st = system("msgmerge ".($quiet?'-q':'')." -s $str_file $tmpfile2 -o - | msgattrib --no-obsolete -o $str_file");
374 error_normal "Text extraction failed: $xgettext: $!\n", undef;
375 error_additional "Will not run msgmerge\n", undef;
377 unlink $tmpfile1 || warn_normal "$tmpfile1: unlink failed: $!\n", undef;
378 unlink $tmpfile2 || warn_normal "$tmpfile2: unlink failed: $!\n", undef;
380 } elsif ($action eq 'update') {
381 my($tmph1, $tmpfile1) = tmpnam();
382 my($tmph2, $tmpfile2) = tmpnam();
383 close $tmph2; # We just want a name
384 # Generate the temporary file that acts as <MODULE>/POTFILES.in
385 for my $input (@in_files) {
386 print $tmph1 "$input\n";
389 # Generate the temporary file that acts as <MODULE>/<LANG>.pot
390 $st = system($xgettext, '-s', '-f', $tmpfile1, '-o', $tmpfile2,
392 (defined $charset_in? ('-I', $charset_in): ()),
393 (defined $charset_out? ('-O', $charset_out): ()));
395 # Merge the temporary "pot file" with the specified po file ($str_file)
396 # FIXME: msgmerge(1) is a Unix dependency
397 # FIXME: need to check the return value
399 my ($tmph3, $tmpfile3) = tmpnam();
400 $st = system("msgcat $str_file $tmpfile2 > $tmpfile3");
401 $st = system("msgmerge ".($quiet?'-q':'')." -s $str_file $tmpfile3 -o - | msgattrib --no-obsolete -o $str_file")
404 $st = system("msgmerge ".($quiet?'-q':'')." -s $str_file $tmpfile2 -o - | msgattrib --no-obsolete -o $str_file");
407 error_normal "Text extraction failed: $xgettext: $!\n", undef;
408 error_additional "Will not run msgmerge\n", undef;
410 unlink $tmpfile1 || warn_normal "$tmpfile1: unlink failed: $!\n", undef;
411 unlink $tmpfile2 || warn_normal "$tmpfile2: unlink failed: $!\n", undef;
413 } elsif ($action eq 'install') {
414 if(!defined($out_dir)) {
415 usage_error("You must specify an output directory when using the install method.");
418 if ( scalar @in_dirs > 1 ) {
419 usage_error("You must specify only one input directory when using the install method.");
422 my $in_dir = shift @in_dirs;
424 if ($in_dir eq $out_dir) {
425 warn "You must specify a different input and output directory.\n";
429 # Make sure the output directory exists
430 # (It will auto-create it, but for compatibility we should not)
431 -d $out_dir || die "$out_dir: The directory does not exist\n";
433 # Try to open the file, because Locale::PO doesn't check :-/
434 open(INPUT, "<$str_file") || die "$str_file: $!\n";
437 # creates the new tmpl file using the new translation
438 for my $input (@in_files) {
439 die "Assertion failed"
440 unless substr($input, 0, length($in_dir) + 1) eq "$in_dir/";
442 my $target = $out_dir . substr($input, length($in_dir));
443 my $targetdir = $` if $target =~ /[^\/]+$/s;
445 if (!defined $type || $input =~ /\.(?:$type)$/) {
446 my $h = TmplTokenizer->new( $input );
447 $h->set_allow_cformat( 1 );
448 VerboseWarnings::set_input_file_name $input;
449 mkdir_recursive($targetdir) unless -d $targetdir;
450 print STDERR "Creating $target...\n" unless $quiet;
451 open( OUTPUT, ">$target" ) || die "$target: $!\n";
452 text_replace( $h, *OUTPUT );
455 # just copying the file
456 mkdir_recursive($targetdir) unless -d $targetdir;
457 system("cp -f $input $target");
458 print STDERR "Copying $input...\n" unless $quiet;
463 usage_error('Unknown action specified.');
467 printf "The %s seems to be successful.\n", $action unless $quiet;
469 printf "%s FAILED.\n", "\u$action" unless $quiet;
473 ###############################################################################
477 ./tmpl_process3.pl [ I<tmpl_process.pl options> ]
481 This is an alternative version of the tmpl_process.pl script,
482 using standard gettext-style PO files. While there still might
483 be changes made to the way it extracts strings, at this moment
484 it should be stable enough for general use; it is already being
485 used for the Chinese and Polish translations.
487 Currently, the create, update, and install actions have all been
488 reimplemented and seem to work.
496 Translation files in standard Uniforum PO format.
497 All standard tools including all gettext tools,
498 plus PO file editors like kbabel(1) etc.
503 Minor changes in whitespace in source templates
504 do not generally require strings to be re-translated.
508 Able to handle <TMPL_VAR> variables in the templates;
509 <TMPL_VAR> variables are usually extracted in proper context,
510 represented by a short %s placeholder.
514 Able to handle text input and radio button INPUT elements
515 in the templates; these INPUT elements are also usually
516 extracted in proper context,
517 represented by a short %S or %p placeholder.
521 Automatic comments in the generated PO files to provide
522 even more context (line numbers, and the names and types
527 The %I<n>$s (or %I<n>$p, etc.) notation can be used
528 for change the ordering of the variables,
529 if such a reordering is required for correct translation.
533 If a particular <TMPL_VAR> should not appear in the
534 translation, it can be suppressed with the %0.0s notation.
538 Using the PO format also means translators can add their
539 own comments in the translation files, if necessary.
543 Create, update, and install actions are all based on the
544 same scanner module. This ensures that update and install
545 have the same idea of what is a translatable string;
546 attribute names in tags, for example, will not be
547 accidentally translated.
553 Anchors are represented by an <AI<n>> notation.
554 The meaning of this non-standard notation might not be obvious.
556 The create action calls xgettext.pl to do the actual work;
557 the update action calls xgettext.pl, msgmerge(1) and msgattrib(1)
558 to do the actual work.
562 xgettext.pl must be present in the current directory; both
563 msgmerge(1) and msgattrib(1) must also be present in the search path.
564 The script currently does not check carefully whether these
565 dependent commands are present.
567 Locale::PO(3) has a lot of bugs. It can neither parse nor
568 generate GNU PO files properly; a couple of workarounds have
569 been written in TmplTokenizer and more is likely to be needed
570 (e.g., to get rid of the "Strange line" warning for #~).
572 This script may not work in Windows.
574 There are probably some other bugs too, since this has not been
585 http://www.saas.nsw.edu.au/koha_wiki/index.php?page=DifficultTerms