2 # This file is part of Koha
3 # Parts copyright 2003-2004 Paul Poulain
4 # Parts copyright 2003-2004 Jerome Vizcaino
5 # Parts copyright 2004 Ambrose Li
9 tmpl_process3.pl - Alternative version of tmpl_process.pl
10 using gettext-compatible translation files
15 #use warnings; FIXME - Bug 2505
18 use File::Temp qw( :POSIX );
20 use VerboseWarnings qw( :warn :die );
22 ###############################################################################
24 use vars qw( @in_files $in_dir $str_file $out_dir $quiet );
25 use vars qw( @excludes $exclude_regex );
26 use vars qw( $recursive_p );
27 use vars qw( $pedantic_p );
29 use vars qw( $type ); # file extension (DOS form without the dot) to match
30 use vars qw( $charset_in $charset_out );
32 ###############################################################################
34 sub find_translation ($) {
38 $key = TmplTokenizer::string_canon($key);
39 $key = TmplTokenizer::charset_convert($key, $charset_in, $charset_out);
40 $key = TmplTokenizer::quote_po($key);
42 return defined $href->{$key}
43 && !$href->{$key}->fuzzy
44 && length Locale::PO->dequote($href->{$key}->msgstr)?
45 Locale::PO->dequote($href->{$key}->msgstr): $s;
48 sub text_replace_tag ($$) {
51 # value [tag=input], meta
52 my $tag = lc($1) if $t =~ /^<(\S+)/s;
54 for my $a ('alt', 'content', 'title', 'value','label') {
56 next if $a eq 'label' && $tag ne 'optgroup';
57 next if $a eq 'content' && $tag ne 'meta';
58 next if $a eq 'value' && ($tag ne 'input'
59 || (ref $attr->{'type'} && $attr->{'type'}->[1] =~ /^(?:checkbox|hidden|radio|text)$/)); # FIXME
60 my($key, $val, $val_orig, $order) = @{$attr->{$a}}; #FIXME
62 my $s = find_translation($val);
63 if ($attr->{$a}->[1] ne $s) { #FIXME
64 $attr->{$a}->[1] = $s; # FIXME
65 $attr->{$a}->[2] = ($s =~ /"/s)? "'$s'": "\"$s\""; #FIXME
74 sprintf(' %s=%s', $_, $attr->{$_}->[2]) #FIXME
76 $attr->{$a}->[3] <=> $attr->{$b}->[3] #FIXME
91 sub text_replace (**) {
94 my $s = TmplTokenizer::next_token $h;
95 last unless defined $s;
96 my($kind, $t, $attr) = ($s->type, $s->string, $s->attributes);
97 if ($kind eq TmplTokenType::TEXT) {
98 print $output find_translation($t);
99 } elsif ($kind eq TmplTokenType::TEXT_PARAMETRIZED) {
100 my $fmt = find_translation($s->form);
101 print $output TmplTokenizer::parametrize($fmt, 1, $s, sub {
103 my($kind, $t, $attr) = ($_->type, $_->string, $_->attributes);
104 $kind == TmplTokenType::TAG && %$attr?
105 text_replace_tag($t, $attr): $t });
106 } elsif ($kind eq TmplTokenType::TAG && %$attr) {
107 print $output text_replace_tag($t, $attr);
108 } elsif ($s->has_js_data) {
109 for my $t (@{$s->js_data}) {
110 # FIXME for this whole block
112 printf $output "%s%s%s", $t->[2], find_translation $t->[3],
115 print $output $t->[1];
118 } elsif (defined $t) {
119 # Quick fix to bug 4472
120 $t = "<!DOCTYPE stylesheet [" if $t =~ /DOCTYPE stylesheet/ ;
126 sub listfiles ($$$) {
127 my($dir, $type, $action) = @_;
129 if (opendir(DIR, $dir)) {
130 my @dirent = readdir DIR; # because DIR is shared when recursing
132 for my $dirent (@dirent) {
133 my $path = "$dir/$dirent";
134 if ($dirent =~ /^\./ || $dirent eq 'CVS' || $dirent eq 'RCS'
135 || (defined $exclude_regex && $dirent =~ /^(?:$exclude_regex)$/)) {
138 push @it, $path if (!defined $type || $dirent =~ /\.(?:$type)$/) || $action eq 'install';
139 } elsif (-d $path && $recursive_p) {
140 push @it, listfiles($path, $type, $action);
144 warn_normal "$dir: $!", undef;
149 ###############################################################################
151 sub mkdir_recursive ($) {
153 local($`, $&, $', $1);
154 $dir = $` if $dir ne /^\/+$/ && $dir =~ /\/+$/;
155 my ($prefix, $basename) = ($dir =~ /\/([^\/]+)$/s)? ($`, $1): ('.', $dir);
156 mkdir_recursive($prefix) if $prefix ne '.' && !-d $prefix;
158 print STDERR "Making directory $dir..." unless $quiet;
159 # creates with rwxrwxr-x permissions
160 mkdir($dir, 0775) || warn_normal "$dir: $!", undef;
164 ###############################################################################
168 my $h = $exitcode? *STDERR: *STDOUT;
170 Usage: $0 create [OPTION]
171 or: $0 update [OPTION]
172 or: $0 install [OPTION]
174 Create or update PO files from templates, or install translated templates.
176 -i, --input=SOURCE Get or update strings from SOURCE file.
177 SOURCE is a directory if -r is also specified.
178 -o, --outputdir=DIRECTORY Install translation(s) to specified DIRECTORY
179 --pedantic-warnings Issue warnings even for detected problems
180 which are likely to be harmless
181 -r, --recursive SOURCE in the -i option is a directory
182 -s, --str-file=FILE Specify FILE as the translation (po) file
183 for input (install) or output (create, update)
184 -x, --exclude=REGEXP Exclude files matching the given REGEXP
185 --help Display this help and exit
186 -q, --quiet no output to screen (except for errors)
188 The -o option is ignored for the "create" and "update" actions.
189 Try `perldoc $0 for perhaps more information.
194 ###############################################################################
196 sub usage_error (;$) {
197 for my $msg (split(/\n/, $_[0])) {
198 print STDERR "$msg\n";
200 print STDERR "Try `$0 --help for more information.\n";
204 ###############################################################################
207 'input|i=s' => \@in_files,
208 'outputdir|o=s' => \$out_dir,
209 'recursive|r' => \$recursive_p,
210 'str-file|s=s' => \$str_file,
211 'exclude|x=s' => \@excludes,
212 'quiet|q' => \$quiet,
213 'pedantic-warnings|pedantic' => sub { $pedantic_p = 1 },
217 VerboseWarnings::set_application_name $0;
218 VerboseWarnings::set_pedantic_mode $pedantic_p;
220 # keep the buggy Locale::PO quiet if it says stupid things
221 $SIG{__WARN__} = sub {
223 print STDERR $s unless $s =~ /^Strange line in [^:]+: #~/s
226 my $action = shift or usage_error('You must specify an ACTION.');
227 usage_error('You must at least specify input and string list filenames.')
228 if !@in_files || !defined $str_file;
230 # Type match defaults to *.tt plus *.inc if not specified
231 $type = "tt|inc|xsl" if !defined($type);
233 # Check the inputs for being files or directories
234 for my $input (@in_files) {
235 usage_error("$input: Input must be a file or directory.\n"
236 . "(Symbolic links are not supported at the moment)")
237 unless -d $input || -f $input;;
240 # Generates the global exclude regular expression
241 $exclude_regex = '(?:'.join('|', @excludes).')' if @excludes;
243 # Generate the list of input files if a directory is specified
244 if (-d $in_files[0]) {
245 die "If you specify a directory as input, you must specify only it.\n"
248 # input is a directory, generates list of files to process
249 $in_dir = $in_files[0];
250 $in_dir =~ s/\/$//; # strips the trailing / if any
251 @in_files = listfiles($in_dir, $type, $action);
253 for my $input (@in_files) {
254 die "You cannot specify input files and directories at the same time.\n"
259 # restores the string list from file
260 $href = Locale::PO->load_file_ashash($str_file);
262 # guess the charsets. HTML::Templates defaults to iso-8859-1
264 die "$str_file: PO file is corrupted, or not a PO file\n" unless defined $href->{'""'};
265 $charset_out = TmplTokenizer::charset_canon $2 if $href->{'""'}->msgstr =~ /\bcharset=(["']?)([^;\s"'\\]+)\1/;
266 $charset_in = $charset_out;
267 warn "Charset in/out: ".$charset_out;
268 # for my $msgid (keys %$href) {
269 # if ($msgid =~ /\bcharset=(["']?)([^;\s"'\\]+)\1/) {
270 # my $candidate = TmplTokenizer::charset_canon $2;
271 # die "Conflicting charsets in msgid: $charset_in vs $candidate => $msgid\n"
272 # if defined $charset_in && $charset_in ne $candidate;
273 # $charset_in = $candidate;
278 # set our charset in to UTF-8
279 if (!defined $charset_in) {
280 $charset_in = TmplTokenizer::charset_canon 'UTF-8';
281 warn "Warning: Can't determine original templates' charset, defaulting to $charset_in\n";
283 # set our charset out to UTF-8
284 if (!defined $charset_out) {
285 $charset_out = TmplTokenizer::charset_canon 'UTF-8';
286 warn "Warning: Charset Out defaulting to $charset_out\n";
288 my $xgettext = './xgettext.pl'; # actual text extractor script
291 if ($action eq 'create') {
292 # updates the list. As the list is empty, every entry will be added
294 warn "Removing empty file $str_file\n";
295 unlink $str_file || die "$str_file: $!\n";
297 die "$str_file: Output file already exists\n" if -f $str_file;
298 my($tmph1, $tmpfile1) = tmpnam();
299 my($tmph2, $tmpfile2) = tmpnam();
300 close $tmph2; # We just want a name
301 # Generate the temporary file that acts as <MODULE>/POTFILES.in
302 for my $input (@in_files) {
303 print $tmph1 "$input\n";
306 warn "I $charset_in O $charset_out";
307 # Generate the specified po file ($str_file)
308 $st = system ($xgettext, '-s', '-f', $tmpfile1, '-o', $tmpfile2,
309 (defined $charset_in? ('-I', $charset_in): ()),
310 (defined $charset_out? ('-O', $charset_out): ())
312 # Run msgmerge so that the pot file looks like a real pot file
313 # We need to help msgmerge a bit by pre-creating a dummy po file that has
314 # the headers and the "" msgid & msgstr. It will fill in the rest.
316 # Merge the temporary "pot file" with the specified po file ($str_file)
317 # FIXME: msgmerge(1) is a Unix dependency
318 # FIXME: need to check the return value
319 unless (-f $str_file) {
320 local(*INPUT, *OUTPUT);
321 open(INPUT, "<$tmpfile2");
322 open(OUTPUT, ">$str_file");
330 $st = system('msgmerge', '-U', '-s', $str_file, $tmpfile2);
332 error_normal "Text extraction failed: $xgettext: $!\n", undef;
333 error_additional "Will not run msgmerge\n", undef;
335 # unlink $tmpfile1 || warn_normal "$tmpfile1: unlink failed: $!\n", undef;
336 # unlink $tmpfile2 || warn_normal "$tmpfile2: unlink failed: $!\n", undef;
338 } elsif ($action eq 'update') {
339 my($tmph1, $tmpfile1) = tmpnam();
340 my($tmph2, $tmpfile2) = tmpnam();
341 close $tmph2; # We just want a name
342 # Generate the temporary file that acts as <MODULE>/POTFILES.in
343 for my $input (@in_files) {
344 print $tmph1 "$input\n";
347 # Generate the temporary file that acts as <MODULE>/<LANG>.pot
348 $st = system($xgettext, '-s', '-f', $tmpfile1, '-o', $tmpfile2,
350 (defined $charset_in? ('-I', $charset_in): ()),
351 (defined $charset_out? ('-O', $charset_out): ()));
353 # Merge the temporary "pot file" with the specified po file ($str_file)
354 # FIXME: msgmerge(1) is a Unix dependency
355 # FIXME: need to check the return value
356 $st = system('msgmerge', '-U', '-s', $str_file, $tmpfile2);
358 error_normal "Text extraction failed: $xgettext: $!\n", undef;
359 error_additional "Will not run msgmerge\n", undef;
361 # unlink $tmpfile1 || warn_normal "$tmpfile1: unlink failed: $!\n", undef;
362 # unlink $tmpfile2 || warn_normal "$tmpfile2: unlink failed: $!\n", undef;
364 } elsif ($action eq 'install') {
365 if(!defined($out_dir)) {
366 usage_error("You must specify an output directory when using the install method.");
369 if ($in_dir eq $out_dir) {
370 warn "You must specify a different input and output directory.\n";
374 # Make sure the output directory exists
375 # (It will auto-create it, but for compatibility we should not)
376 -d $out_dir || die "$out_dir: The directory does not exist\n";
378 # Try to open the file, because Locale::PO doesn't check :-/
379 open(INPUT, "<$str_file") || die "$str_file: $!\n";
382 # creates the new tmpl file using the new translation
383 for my $input (@in_files) {
384 die "Assertion failed"
385 unless substr($input, 0, length($in_dir) + 1) eq "$in_dir/";
386 # print "$input / $type\n";
387 if (!defined $type || $input =~ /\.(?:$type)$/) {
388 my $h = TmplTokenizer->new( $input );
389 $h->set_allow_cformat( 1 );
390 VerboseWarnings::set_input_file_name $input;
392 my $target = $out_dir . substr($input, length($in_dir));
393 my $targetdir = $` if $target =~ /[^\/]+$/s;
394 mkdir_recursive($targetdir) unless -d $targetdir;
395 print STDERR "Creating $target...\n" unless $quiet;
396 open( OUTPUT, ">$target" ) || die "$target: $!\n";
397 binmode( OUTPUT, ":utf8" );
398 text_replace( $h, *OUTPUT );
401 # just copying the file
402 my $target = $out_dir . substr($input, length($in_dir));
403 my $targetdir = $` if $target =~ /[^\/]+$/s;
404 mkdir_recursive($targetdir) unless -d $targetdir;
405 system("cp -f $input $target");
406 print STDERR "Copying $input...\n" unless $quiet;
411 usage_error('Unknown action specified.');
415 printf "The %s seems to be successful.\n", $action unless $quiet;
417 printf "%s FAILED.\n", "\u$action" unless $quiet;
421 ###############################################################################
425 ./tmpl_process3.pl [ I<tmpl_process.pl options> ]
429 This is an alternative version of the tmpl_process.pl script,
430 using standard gettext-style PO files. While there still might
431 be changes made to the way it extracts strings, at this moment
432 it should be stable enough for general use; it is already being
433 used for the Chinese and Polish translations.
435 Currently, the create, update, and install actions have all been
436 reimplemented and seem to work.
444 Translation files in standard Uniforum PO format.
445 All standard tools including all gettext tools,
446 plus PO file editors like kbabel(1) etc.
451 Minor changes in whitespace in source templates
452 do not generally require strings to be re-translated.
456 Able to handle <TMPL_VAR> variables in the templates;
457 <TMPL_VAR> variables are usually extracted in proper context,
458 represented by a short %s placeholder.
462 Able to handle text input and radio button INPUT elements
463 in the templates; these INPUT elements are also usually
464 extracted in proper context,
465 represented by a short %S or %p placeholder.
469 Automatic comments in the generated PO files to provide
470 even more context (line numbers, and the names and types
475 The %I<n>$s (or %I<n>$p, etc.) notation can be used
476 for change the ordering of the variables,
477 if such a reordering is required for correct translation.
481 If a particular <TMPL_VAR> should not appear in the
482 translation, it can be suppressed with the %0.0s notation.
486 Using the PO format also means translators can add their
487 own comments in the translation files, if necessary.
491 Create, update, and install actions are all based on the
492 same scanner module. This ensures that update and install
493 have the same idea of what is a translatable string;
494 attribute names in tags, for example, will not be
495 accidentally translated.
501 Anchors are represented by an <AI<n>> notation.
502 The meaning of this non-standard notation might not be obvious.
504 The create action calls xgettext.pl to do the actual work;
505 the update action calls xgettext.pl and msgmerge(1) to do the
510 xgettext.pl must be present in the current directory; the
511 msgmerge(1) command must also be present in the search path.
512 The script currently does not check carefully whether these
513 dependent commands are present.
515 Locale::PO(3) has a lot of bugs. It can neither parse nor
516 generate GNU PO files properly; a couple of workarounds have
517 been written in TmplTokenizer and more is likely to be needed
518 (e.g., to get rid of the "Strange line" warning for #~).
520 This script may not work in Windows.
522 There are probably some other bugs too, since this has not been
533 http://www.saas.nsw.edu.au/koha_wiki/index.php?page=DifficultTerms