2 # This file is part of Koha
3 # Parts copyright 2003-2004 Paul Poulain
4 # Parts copyright 2003-2004 Jerome Vizcaino
5 # Parts copyright 2004 Ambrose Li
9 tmpl_process3.pl - Alternative version of tmpl_process.pl
10 using gettext-compatible translation files
17 use File::Temp qw( :POSIX );
19 use VerboseWarnings qw( :warn :die );
21 ###############################################################################
23 use vars qw( @in_files $in_dir $str_file $out_dir );
24 use vars qw( @excludes $exclude_regex );
25 use vars qw( $recursive_p );
26 use vars qw( $pedantic_p );
28 use vars qw( $type ); # file extension (DOS form without the dot) to match
29 use vars qw( $charset_in $charset_out );
31 ###############################################################################
33 sub find_translation ($) {
37 $key = TmplTokenizer::string_canon($key);
38 $key = TmplTokenizer::charset_convert($key, $charset_in, $charset_out);
39 $key = TmplTokenizer::quote_po($key);
41 return defined $href->{$key}
42 && !$href->{$key}->fuzzy
43 && length Locale::PO->dequote($href->{$key}->msgstr)?
44 Locale::PO->dequote($href->{$key}->msgstr): $s;
47 sub text_replace_tag ($$) {
50 # value [tag=input], meta
51 my $tag = lc($1) if $t =~ /^<(\S+)/s;
53 for my $a ('alt', 'content', 'title', 'value') {
55 next if $a eq 'content' && $tag ne 'meta';
56 next if $a eq 'value' && ($tag ne 'input'
57 || (ref $attr->{'type'} && $attr->{'type'}->[1] =~ /^(?:hidden|radio)$/)); # FIXME
58 my($key, $val, $val_orig, $order) = @{$attr->{$a}}; #FIXME
60 my $s = find_translation($val);
61 if ($attr->{$a}->[1] ne $s) { #FIXME
62 $attr->{$a}->[1] = $s; # FIXME
63 $attr->{$a}->[2] = ($s =~ /"/s)? "'$s'": "\"$s\""; #FIXME
72 sprintf(' %s=%s', $_, $attr->{$_}->[2]) #FIXME
74 $attr->{$a}->[3] <=> $attr->{$b}->[3] #FIXME
83 sub text_replace (**) {
86 my $s = TmplTokenizer::next_token $h;
87 last unless defined $s;
88 my($kind, $t, $attr) = ($s->type, $s->string, $s->attributes);
89 if ($kind eq TmplTokenType::TEXT) {
90 print $output find_translation($t);
91 } elsif ($kind eq TmplTokenType::TEXT_PARAMETRIZED) {
92 my $fmt = find_translation($s->form);
93 print $output TmplTokenizer::parametrize($fmt, 1, $s, sub {
95 my($kind, $t, $attr) = ($_->type, $_->string, $_->attributes);
96 $kind == TmplTokenType::TAG && %$attr?
97 text_replace_tag($t, $attr): $t });
98 } elsif ($kind eq TmplTokenType::TAG && %$attr) {
99 print $output text_replace_tag($t, $attr);
100 } elsif (defined $t) {
107 my($dir, $type) = @_;
109 if (opendir(DIR, $dir)) {
110 my @dirent = readdir DIR; # because DIR is shared when recursing
112 for my $dirent (@dirent) {
113 my $path = "$dir/$dirent";
114 if ($dirent =~ /^\./ || $dirent eq 'CVS' || $dirent eq 'RCS'
115 || (defined $exclude_regex && $dirent =~ /^(?:$exclude_regex)$/)) {
118 push @it, $path if !defined $type || $dirent =~ /\.(?:$type)$/;
119 } elsif (-d $path && $recursive_p) {
120 push @it, listfiles($path, $type);
124 warn_normal "$dir: $!", undef;
129 ###############################################################################
133 my $h = $exitcode? *STDERR: *STDOUT;
135 Usage: $0 create [OPTION]
136 or: $0 update [OPTION]
137 or: $0 install [OPTION]
139 Create or update PO files from templates, or install translated templates.
141 -i, --input=SOURCE Get or update strings from SOURCE file.
142 SOURCE is a directory if -r is also specified.
143 -o, --outputdir=DIRECTORY Install translation(s) to specified DIRECTORY
144 --pedantic-warnings Issue warnings even for detected problems
145 which are likely to be harmless
146 -r, --recursive SOURCE in the -i option is a directory
147 -s, --str-file=FILE Specify FILE as the translation (po) file
148 for input (install) or output (create, update)
149 -x, --exclude=REGEXP Exclude files matching the given REGEXP
150 --help Display this help and exit
152 The -o option is ignored for the "create" and "update" actions.
153 Try `perldoc $0' for perhaps more information.
158 ###############################################################################
160 sub usage_error (;$) {
161 for my $msg (split(/\n/, $_[0])) {
162 print STDERR "$msg\n";
164 print STDERR "Try `$0 --help' for more information.\n";
168 ###############################################################################
171 'input|i=s' => \@in_files,
172 'outputdir|o=s' => \$out_dir,
173 'recursive|r' => \$recursive_p,
174 'str-file|s=s' => \$str_file,
175 'exclude|x=s' => \@excludes,
176 'pedantic-warnings|pedantic' => sub { $pedantic_p = 1 },
180 VerboseWarnings::set_application_name $0;
181 VerboseWarnings::set_pedantic_mode $pedantic_p;
183 # keep the buggy Locale::PO quiet if it says stupid things
184 $SIG{__WARN__} = sub {
186 print STDERR $s unless $s =~ /^Strange line in [^:]+: #~/s
189 my $action = shift or usage_error('You must specify an ACTION.');
190 usage_error('You must at least specify input and string list filenames.')
191 if !@in_files || !defined $str_file;
193 # Type match defaults to *.tmpl plus *.inc if not specified
194 $type = "tmpl|inc" if !defined($type);
196 # Check the inputs for being files or directories
197 for my $input (@in_files) {
198 usage_error("$input: Input must be a file or directory.\n"
199 . "(Symbolic links are not supported at the moment)")
200 unless -d $input || -f $input;;
203 # Generates the global exclude regular expression
204 $exclude_regex = '(?:'.join('|', @excludes).')' if @excludes;
206 # Generate the list of input files if a directory is specified
207 if (-d $in_files[0]) {
208 die "If you specify a directory as input, you must specify only it.\n"
211 # input is a directory, generates list of files to process
212 $in_dir = $in_files[0];
213 $in_dir =~ s/\/$//; # strips the trailing / if any
214 @in_files = listfiles($in_dir, $type);
216 for my $input (@in_files) {
217 die "You cannot specify input files and directories at the same time.\n"
222 # restores the string list from file
223 $href = Locale::PO->load_file_ashash($str_file);
225 # guess the charsets. HTML::Templates defaults to iso-8859-1
227 die "$str_file: PO file is corrupted, or not a PO file\n"
228 unless defined $href->{'""'};
229 $charset_out = TmplTokenizer::charset_canon $2
230 if $href->{'""'}->msgstr =~ /\bcharset=(["']?)([^;\s"'\\]+)\1/;
231 for my $msgid (keys %$href) {
232 if ($msgid =~ /\bcharset=(["']?)([^;\s"'\\]+)\1/) {
233 my $candidate = TmplTokenizer::charset_canon $2;
234 die "Conflicting charsets in msgid: $charset_in vs $candidate\n"
235 if defined $charset_in && $charset_in ne $candidate;
236 $charset_in = $candidate;
240 if (!defined $charset_in) {
241 $charset_in = TmplTokenizer::charset_canon 'iso8859-1';
242 warn "Warning: Can't determine original templates' charset, defaulting to $charset_in\n";
245 my $xgettext = './xgettext.pl'; # actual text extractor script
248 if ($action eq 'create') {
249 # updates the list. As the list is empty, every entry will be added
251 warn "Removing empty file $str_file\n";
252 unlink $str_file || die "$str_file: $!\n";
254 die "$str_file: Output file already exists\n" if -f $str_file;
255 my($tmph, $tmpfile) = tmpnam();
256 # Generate the temporary file that acts as <MODULE>/POTFILES.in
257 for my $input (@in_files) {
258 print $tmph "$input\n";
261 # Generate the specified po file ($str_file)
262 $st = system ($xgettext, '-s', '-f', $tmpfile, '-o', $str_file);
263 warn_normal "Text extraction failed: $xgettext: $!\n", undef if $st != 0;
264 # unlink $tmpfile || warn_normal "$tmpfile: unlink failed: $!\n", undef;
266 } elsif ($action eq 'update') {
267 my($tmph1, $tmpfile1) = tmpnam();
268 my($tmph2, $tmpfile2) = tmpnam();
269 close $tmph2; # We just want a name
270 # Generate the temporary file that acts as <MODULE>/POTFILES.in
271 for my $input (@in_files) {
272 print $tmph1 "$input\n";
275 # Generate the temporary file that acts as <MODULE>/<LANG>.pot
276 $st = system($xgettext, '-s', '-f', $tmpfile1, '-o', $tmpfile2,
278 (defined $charset_in? ('-I', $charset_in): ()),
279 (defined $charset_out? ('-O', $charset_out): ()));
281 # Merge the temporary "pot file" with the specified po file ($str_file)
282 # FIXME: msgmerge(1) is a Unix dependency
283 # FIXME: need to check the return value
284 $st = system('msgmerge', '-U', '-s', $str_file, $tmpfile2);
286 error_normal "Text extraction failed: $xgettext: $!\n", undef;
287 error_additional "Will not run msgmerge\n", undef;
289 # unlink $tmpfile1 || warn_normal "$tmpfile1: unlink failed: $!\n", undef;
290 # unlink $tmpfile2 || warn_normal "$tmpfile2: unlink failed: $!\n", undef;
292 } elsif ($action eq 'install') {
293 if(!defined($out_dir)) {
294 usage_error("You must specify an output directory when using the install method.");
297 if ($in_dir eq $out_dir) {
298 warn "You must specify a different input and output directory.\n";
302 # Make sure the output directory exists
303 # (It will auto-create it, but for compatibility we should not)
304 -d $out_dir || die "$out_dir: The directory does not exist\n";
306 # Try to open the file, because Locale::PO doesn't check :-/
307 open(INPUT, "<$str_file") || die "$str_file: $!\n";
310 # creates the new tmpl file using the new translation
311 for my $input (@in_files) {
312 die "Assertion failed"
313 unless substr($input, 0, length($in_dir) + 1) eq "$in_dir/";
315 my $h = TmplTokenizer->new( $input );
316 $h->set_allow_cformat( 1 );
317 VerboseWarnings::set_input_file_name $input;
319 my $target = $out_dir . substr($input, length($in_dir));
320 my $targetdir = $` if $target =~ /[^\/]+$/s;
321 if (!-d $targetdir) {
322 print STDERR "Making directory $targetdir...";
323 # creates with rwxrwxr-x permissions
324 mkdir($targetdir, 0775) || warn_normal "$targetdir: $!", undef;
326 print STDERR "Creating $target...\n";
327 open( OUTPUT, ">$target" ) || die "$target: $!\n";
328 text_replace( $h, *OUTPUT );
333 usage_error('Unknown action specified.');
337 printf "The %s seems to be successful.\n", $action;
339 printf "%s FAILED.\n", "\u$action";
343 ###############################################################################
347 ./tmpl_process3.pl [ I<tmpl_process.pl options> ]
351 This is an alternative version of the tmpl_process.pl script,
352 using standard gettext-style PO files. While there still might
353 be changes made to the way it extracts strings, at this moment
354 it should be stable enough for general use; it is already being
355 used for the Chinese and Polish translations.
357 Currently, the create, update, and install actions have all been
358 reimplemented and seem to work.
366 Translation files in standard Uniforum PO format.
367 All standard tools including all gettext tools,
368 plus PO file editors like kbabel(1) etc.
373 Minor changes in whitespace in source templates
374 do not generally require strings to be re-translated.
378 Able to handle <TMPL_VAR> variables in the templates;
379 <TMPL_VAR> variables are usually extracted in proper context,
380 represented by a short %s placeholder.
384 Able to handle text input and radio button INPUT elements
385 in the templates; these INPUT elements are also usually
386 extracted in proper context,
387 represented by a short %S or %p placeholder.
391 Automatic comments in the generated PO files to provide
392 even more context (line numbers, and the names and types
397 The %I<n>$s (or %I<n>$p, etc.) notation can be used
398 for change the ordering of the variables,
399 if such a reordering is required for correct translation.
403 If a particular <TMPL_VAR> should not appear in the
404 translation, it can be suppressed with the %0.0s notation.
408 Using the PO format also means translators can add their
409 own comments in the translation files, if necessary.
413 Create, update, and install actions are all based on the
414 same scanner module. This ensures that update and install
415 have the same idea of what is a translatable string;
416 attribute names in tags, for example, will not be
417 accidentally translated.
423 Anchors are represented by an <AI<n>> notation.
424 The meaning of this non-standard notation might not be obvious.
426 The create action calls xgettext.pl to do the actual work;
427 the update action calls xgettext.pl and msgmerge(1) to do the
432 xgettext.pl must be present in the current directory; the
433 msgmerge(1) command must also be present in the search path.
434 The script currently does not check carefully whether these
435 dependent commands are present.
437 Locale::PO(3) has a lot of bugs. It can neither parse nor
438 generate GNU PO files properly; a couple of workarounds have
439 been written in TmplTokenizer and more is likely to be needed
440 (e.g., to get rid of the "Strange line" warning for #~).
442 This script may not work in Windows.
444 There are probably some other bugs too, since this has not been
455 http://www.saas.nsw.edu.au/koha_wiki/index.php?page=DifficultTerms