2 # This file is part of Koha
3 # Parts copyright 2003-2004 Paul Poulain
4 # Parts copyright 2003-2004 Jerome Vizcaino
5 # Parts copyright 2004 Ambrose Li
9 tmpl_process3.pl - Alternative version of tmpl_process.pl
10 using gettext-compatible translation files
17 use File::Temp qw( :POSIX );
19 use VerboseWarnings qw( :warn :die );
21 ###############################################################################
23 use vars qw( @in_files $in_dir $str_file $out_dir $quiet );
24 use vars qw( @excludes $exclude_regex );
25 use vars qw( $recursive_p );
26 use vars qw( $pedantic_p );
28 use vars qw( $type ); # file extension (DOS form without the dot) to match
29 use vars qw( $charset_in $charset_out );
31 ###############################################################################
33 sub find_translation ($) {
37 $key = TmplTokenizer::string_canon($key);
38 $key = TmplTokenizer::charset_convert($key, $charset_in, $charset_out);
39 $key = TmplTokenizer::quote_po($key);
41 return defined $href->{$key}
42 && !$href->{$key}->fuzzy
43 && length Locale::PO->dequote($href->{$key}->msgstr)?
44 Locale::PO->dequote($href->{$key}->msgstr): $s;
47 sub text_replace_tag ($$) {
50 # value [tag=input], meta
51 my $tag = lc($1) if $t =~ /^<(\S+)/s;
53 for my $a ('alt', 'content', 'title', 'value') {
55 next if $a eq 'content' && $tag ne 'meta';
56 next if $a eq 'value' && ($tag ne 'input'
57 || (ref $attr->{'type'} && $attr->{'type'}->[1] =~ /^(?:hidden|radio|text)$/)); # FIXME
58 my($key, $val, $val_orig, $order) = @{$attr->{$a}}; #FIXME
60 my $s = find_translation($val);
61 if ($attr->{$a}->[1] ne $s) { #FIXME
62 $attr->{$a}->[1] = $s; # FIXME
63 $attr->{$a}->[2] = ($s =~ /"/s)? "'$s'": "\"$s\""; #FIXME
72 sprintf(' %s=%s', $_, $attr->{$_}->[2]) #FIXME
74 $attr->{$a}->[3] <=> $attr->{$b}->[3] #FIXME
83 sub text_replace (**) {
86 my $s = TmplTokenizer::next_token $h;
87 last unless defined $s;
88 my($kind, $t, $attr) = ($s->type, $s->string, $s->attributes);
89 if ($kind eq TmplTokenType::TEXT) {
90 print $output find_translation($t);
91 } elsif ($kind eq TmplTokenType::TEXT_PARAMETRIZED) {
92 my $fmt = find_translation($s->form);
93 print $output TmplTokenizer::parametrize($fmt, 1, $s, sub {
95 my($kind, $t, $attr) = ($_->type, $_->string, $_->attributes);
96 $kind == TmplTokenType::TAG && %$attr?
97 text_replace_tag($t, $attr): $t });
98 } elsif ($kind eq TmplTokenType::TAG && %$attr) {
99 print $output text_replace_tag($t, $attr);
100 } elsif ($s->has_js_data) {
101 for my $t (@{$s->js_data}) {
102 # FIXME for this whole block
104 printf $output "%s%s%s", $t->[2], find_translation $t->[3],
107 print $output $t->[1];
110 } elsif (defined $t) {
116 sub listfiles ($$$) {
117 my($dir, $type, $action) = @_;
119 if (opendir(DIR, $dir)) {
120 my @dirent = readdir DIR; # because DIR is shared when recursing
122 for my $dirent (@dirent) {
123 my $path = "$dir/$dirent";
124 if ($dirent =~ /^\./ || $dirent eq 'CVS' || $dirent eq 'RCS'
125 || (defined $exclude_regex && $dirent =~ /^(?:$exclude_regex)$/)) {
128 push @it, $path if (!defined $type || $dirent =~ /\.(?:$type)$/) || $action eq 'install';
129 } elsif (-d $path && $recursive_p) {
130 push @it, listfiles($path, $type, $action);
134 warn_normal "$dir: $!", undef;
139 ###############################################################################
141 sub mkdir_recursive ($) {
143 local($`, $&, $', $1);
144 $dir = $` if $dir ne /^\/+$/ && $dir =~ /\/+$/;
145 my ($prefix, $basename) = ($dir =~ /\/([^\/]+)$/s)? ($`, $1): ('.', $dir);
146 mkdir_recursive($prefix) if $prefix ne '.' && !-d $prefix;
148 print STDERR "Making directory $dir..." unless $quiet;
149 # creates with rwxrwxr-x permissions
150 mkdir($dir, 0775) || warn_normal "$dir: $!", undef;
154 ###############################################################################
158 my $h = $exitcode? *STDERR: *STDOUT;
160 Usage: $0 create [OPTION]
161 or: $0 update [OPTION]
162 or: $0 install [OPTION]
164 Create or update PO files from templates, or install translated templates.
166 -i, --input=SOURCE Get or update strings from SOURCE file.
167 SOURCE is a directory if -r is also specified.
168 -o, --outputdir=DIRECTORY Install translation(s) to specified DIRECTORY
169 --pedantic-warnings Issue warnings even for detected problems
170 which are likely to be harmless
171 -r, --recursive SOURCE in the -i option is a directory
172 -s, --str-file=FILE Specify FILE as the translation (po) file
173 for input (install) or output (create, update)
174 -x, --exclude=REGEXP Exclude files matching the given REGEXP
175 --help Display this help and exit
176 -q, --quiet no output to screen (except for errors)
178 The -o option is ignored for the "create" and "update" actions.
179 Try `perldoc $0 for perhaps more information.
184 ###############################################################################
186 sub usage_error (;$) {
187 for my $msg (split(/\n/, $_[0])) {
188 print STDERR "$msg\n";
190 print STDERR "Try `$0 --help for more information.\n";
194 ###############################################################################
197 'input|i=s' => \@in_files,
198 'outputdir|o=s' => \$out_dir,
199 'recursive|r' => \$recursive_p,
200 'str-file|s=s' => \$str_file,
201 'exclude|x=s' => \@excludes,
202 'quiet|q' => \$quiet,
203 'pedantic-warnings|pedantic' => sub { $pedantic_p = 1 },
207 VerboseWarnings::set_application_name $0;
208 VerboseWarnings::set_pedantic_mode $pedantic_p;
210 # keep the buggy Locale::PO quiet if it says stupid things
211 $SIG{__WARN__} = sub {
213 print STDERR $s unless $s =~ /^Strange line in [^:]+: #~/s
216 my $action = shift or usage_error('You must specify an ACTION.');
217 usage_error('You must at least specify input and string list filenames.')
218 if !@in_files || !defined $str_file;
220 # Type match defaults to *.tmpl plus *.inc if not specified
221 $type = "tmpl|inc" if !defined($type);
223 # Check the inputs for being files or directories
224 for my $input (@in_files) {
225 usage_error("$input: Input must be a file or directory.\n"
226 . "(Symbolic links are not supported at the moment)")
227 unless -d $input || -f $input;;
230 # Generates the global exclude regular expression
231 $exclude_regex = '(?:'.join('|', @excludes).')' if @excludes;
233 # Generate the list of input files if a directory is specified
234 if (-d $in_files[0]) {
235 die "If you specify a directory as input, you must specify only it.\n"
238 # input is a directory, generates list of files to process
239 $in_dir = $in_files[0];
240 $in_dir =~ s/\/$//; # strips the trailing / if any
241 @in_files = listfiles($in_dir, $type, $action);
243 for my $input (@in_files) {
244 die "You cannot specify input files and directories at the same time.\n"
249 # restores the string list from file
250 $href = Locale::PO->load_file_ashash($str_file);
252 # guess the charsets. HTML::Templates defaults to iso-8859-1
254 die "$str_file: PO file is corrupted, or not a PO file\n"
255 unless defined $href->{'""'};
256 $charset_out = TmplTokenizer::charset_canon $2
257 if $href->{'""'}->msgstr =~ /\bcharset=(["']?)([^;\s"'\\]+)\1/;
258 $charset_in = 'UTF-8';
259 warn "Charset out: ".$charset_out;
260 # for my $msgid (keys %$href) {
261 # if ($msgid =~ /\bcharset=(["']?)([^;\s"'\\]+)\1/) {
262 # my $candidate = TmplTokenizer::charset_canon $2;
263 # die "Conflicting charsets in msgid: $charset_in vs $candidate => $msgid\n"
264 # if defined $charset_in && $charset_in ne $candidate;
265 # $charset_in = $candidate;
270 # set our charset in to UTF-8
271 if (!defined $charset_in) {
272 $charset_in = TmplTokenizer::charset_canon 'utf-8';
273 warn "Warning: Can't determine original templates' charset, defaulting to $charset_in\n";
275 # set our charset out to UTF-8
276 if (!defined $charset_out) {
277 $charset_out = TmplTokenizer::charset_canon 'utf-8';
278 warn "Warning: Charset Out defaulting to $charset_out\n";
280 my $xgettext = './xgettext.pl'; # actual text extractor script
283 if ($action eq 'create') {
284 # updates the list. As the list is empty, every entry will be added
286 warn "Removing empty file $str_file\n";
287 unlink $str_file || die "$str_file: $!\n";
289 die "$str_file: Output file already exists\n" if -f $str_file;
290 my($tmph1, $tmpfile1) = tmpnam();
291 my($tmph2, $tmpfile2) = tmpnam();
292 close $tmph2; # We just want a name
293 # Generate the temporary file that acts as <MODULE>/POTFILES.in
294 for my $input (@in_files) {
295 print $tmph1 "$input\n";
298 warn "I $charset_in O $charset_out";
299 # Generate the specified po file ($str_file)
300 $st = system ($xgettext, '-s', '-f', $tmpfile1, '-o', $tmpfile2,
301 (defined $charset_in? ('-I', $charset_in): ()),
302 (defined $charset_out? ('-O', $charset_out): ())
304 # Run msgmerge so that the pot file looks like a real pot file
305 # We need to help msgmerge a bit by pre-creating a dummy po file that has
306 # the headers and the "" msgid & msgstr. It will fill in the rest.
308 # Merge the temporary "pot file" with the specified po file ($str_file)
309 # FIXME: msgmerge(1) is a Unix dependency
310 # FIXME: need to check the return value
311 unless (-f $str_file) {
312 local(*INPUT, *OUTPUT);
313 open(INPUT, "<$tmpfile2");
314 open(OUTPUT, ">$str_file");
322 $st = system('msgmerge', '-U', '-s', $str_file, $tmpfile2);
324 error_normal "Text extraction failed: $xgettext: $!\n", undef;
325 error_additional "Will not run msgmerge\n", undef;
327 # unlink $tmpfile1 || warn_normal "$tmpfile1: unlink failed: $!\n", undef;
328 # unlink $tmpfile2 || warn_normal "$tmpfile2: unlink failed: $!\n", undef;
330 } elsif ($action eq 'update') {
331 my($tmph1, $tmpfile1) = tmpnam();
332 my($tmph2, $tmpfile2) = tmpnam();
333 close $tmph2; # We just want a name
334 # Generate the temporary file that acts as <MODULE>/POTFILES.in
335 for my $input (@in_files) {
336 print $tmph1 "$input\n";
339 # Generate the temporary file that acts as <MODULE>/<LANG>.pot
340 $st = system($xgettext, '-s', '-f', $tmpfile1, '-o', $tmpfile2,
342 (defined $charset_in? ('-I', $charset_in): ()),
343 (defined $charset_out? ('-O', $charset_out): ()));
345 # Merge the temporary "pot file" with the specified po file ($str_file)
346 # FIXME: msgmerge(1) is a Unix dependency
347 # FIXME: need to check the return value
348 $st = system('msgmerge', '-U', '-s', $str_file, $tmpfile2);
350 error_normal "Text extraction failed: $xgettext: $!\n", undef;
351 error_additional "Will not run msgmerge\n", undef;
353 # unlink $tmpfile1 || warn_normal "$tmpfile1: unlink failed: $!\n", undef;
354 # unlink $tmpfile2 || warn_normal "$tmpfile2: unlink failed: $!\n", undef;
356 } elsif ($action eq 'install') {
357 if(!defined($out_dir)) {
358 usage_error("You must specify an output directory when using the install method.");
361 if ($in_dir eq $out_dir) {
362 warn "You must specify a different input and output directory.\n";
366 # Make sure the output directory exists
367 # (It will auto-create it, but for compatibility we should not)
368 -d $out_dir || die "$out_dir: The directory does not exist\n";
370 # Try to open the file, because Locale::PO doesn't check :-/
371 open(INPUT, "<$str_file") || die "$str_file: $!\n";
374 # creates the new tmpl file using the new translation
375 for my $input (@in_files) {
376 die "Assertion failed"
377 unless substr($input, 0, length($in_dir) + 1) eq "$in_dir/";
378 # print "$input / $type\n";
379 if (!defined $type || $input =~ /\.(?:$type)$/) {
380 my $h = TmplTokenizer->new( $input );
381 $h->set_allow_cformat( 1 );
382 VerboseWarnings::set_input_file_name $input;
384 my $target = $out_dir . substr($input, length($in_dir));
385 my $targetdir = $` if $target =~ /[^\/]+$/s;
386 mkdir_recursive($targetdir) unless -d $targetdir;
387 print STDERR "Creating $target...\n" unless $quiet;
388 open( OUTPUT, ">$target" ) || die "$target: $!\n";
389 text_replace( $h, *OUTPUT );
392 # just copying the file
393 my $target = $out_dir . substr($input, length($in_dir));
394 my $targetdir = $` if $target =~ /[^\/]+$/s;
395 mkdir_recursive($targetdir) unless -d $targetdir;
396 system("cp -f $input $target");
397 print STDERR "Copying $input...\n" unless $quiet;
402 usage_error('Unknown action specified.');
406 printf "The %s seems to be successful.\n", $action unless $quiet;
408 printf "%s FAILED.\n", "\u$action" unless $quiet;
412 ###############################################################################
416 ./tmpl_process3.pl [ I<tmpl_process.pl options> ]
420 This is an alternative version of the tmpl_process.pl script,
421 using standard gettext-style PO files. While there still might
422 be changes made to the way it extracts strings, at this moment
423 it should be stable enough for general use; it is already being
424 used for the Chinese and Polish translations.
426 Currently, the create, update, and install actions have all been
427 reimplemented and seem to work.
435 Translation files in standard Uniforum PO format.
436 All standard tools including all gettext tools,
437 plus PO file editors like kbabel(1) etc.
442 Minor changes in whitespace in source templates
443 do not generally require strings to be re-translated.
447 Able to handle <TMPL_VAR> variables in the templates;
448 <TMPL_VAR> variables are usually extracted in proper context,
449 represented by a short %s placeholder.
453 Able to handle text input and radio button INPUT elements
454 in the templates; these INPUT elements are also usually
455 extracted in proper context,
456 represented by a short %S or %p placeholder.
460 Automatic comments in the generated PO files to provide
461 even more context (line numbers, and the names and types
466 The %I<n>$s (or %I<n>$p, etc.) notation can be used
467 for change the ordering of the variables,
468 if such a reordering is required for correct translation.
472 If a particular <TMPL_VAR> should not appear in the
473 translation, it can be suppressed with the %0.0s notation.
477 Using the PO format also means translators can add their
478 own comments in the translation files, if necessary.
482 Create, update, and install actions are all based on the
483 same scanner module. This ensures that update and install
484 have the same idea of what is a translatable string;
485 attribute names in tags, for example, will not be
486 accidentally translated.
492 Anchors are represented by an <AI<n>> notation.
493 The meaning of this non-standard notation might not be obvious.
495 The create action calls xgettext.pl to do the actual work;
496 the update action calls xgettext.pl and msgmerge(1) to do the
501 xgettext.pl must be present in the current directory; the
502 msgmerge(1) command must also be present in the search path.
503 The script currently does not check carefully whether these
504 dependent commands are present.
506 Locale::PO(3) has a lot of bugs. It can neither parse nor
507 generate GNU PO files properly; a couple of workarounds have
508 been written in TmplTokenizer and more is likely to be needed
509 (e.g., to get rid of the "Strange line" warning for #~).
511 This script may not work in Windows.
513 There are probably some other bugs too, since this has not been
524 http://www.saas.nsw.edu.au/koha_wiki/index.php?page=DifficultTerms