source: git/lib/po-to-msg.pl @ 6316bfa

RELEASE/1.2debug-cidebug-ci-sanitisersstereowalls-data
Last change on this file since 6316bfa was c7267bc, checked in by Olly Betts <olly@…>, 9 years ago

lib/po-to-msg.pl: Add check that language code in file matches the
filename.

  • Property mode set to 100755
File size: 8.3 KB
Line 
1#!/usr/bin/perl -w
2require 5.008;
3use bytes;
4use strict;
5use Locale::PO;
6
7use integer;
8
9# Magic identifier (12 bytes)
10my $magic = "Svx\nMsg\r\n\xfe\xff\0";
11# Designed to be corrupted by ASCII ftp, top bit stripping (or
12# being used for parity).  Contains a zero byte so more likely
13# to be flagged as data (e.g. by perl's "-B" test).
14
15my $srcdir = $0;
16$srcdir =~ s!/[^/]+$!!;
17
18my $major = 0;
19my $minor = 8;
20
21# File format (multi-byte numbers in network order (bigendian)):
22# 12 bytes: Magic identifier
23# 1 byte:   File format major version number (0)
24# 1 byte:   File format minor version number (8)
25# 2 bytes:  Number of messages (N)
26# 4 bytes:  Offset from XXXX to end of file
27# XXXX:
28# N*:
29# <message> NUL
30
31my %msgs = ();
32${$msgs{'en'}}[0] = '©';
33
34# my %uses = ();
35
36my $file;
37
38my %n = ();
39my %loc = ();
40$file = "$srcdir/survex.pot";
41my $num_list = Locale::PO->load_file_asarray($file);
42foreach my $po_entry (@{$num_list}) {
43    my $ref = $po_entry->reference;
44    (defined $ref && $ref =~ /^n:(\d+)$/m) or next;
45    my $msgno = $1;
46    my $key = $po_entry->msgid;
47    my $msg = c_unescape($po_entry->dequote($key));
48    my $where = $file . ":" . $po_entry->loaded_line_number;
49    ${$loc{'en'}}[$msgno] = $where;
50    if (${$msgs{'en'}}[$msgno]) {
51        print STDERR "$where: warning: already had message $msgno for language 'en'\n";
52    }
53    ${$msgs{'en'}}[$msgno] = $msg;
54    ++$n{$msgno};
55}
56my $last = 0;
57for (sort { $a <=> $b } keys %n) {
58    if ($_ > $last + 1) {
59        print STDERR "$file: Unused msg numbers: ", join(" ", $last + 1 .. $_ - 1), "\n";
60    }
61    $last = $_;
62}
63print STDERR "$file: Last used msg number: $last\n";
64%n = ();
65
66for my $po_file (@ARGV) {
67    my $language = $po_file;
68    $language =~ s/\.po$//;
69
70    $file = "$srcdir/$po_file";
71    my $po_hash = Locale::PO->load_file_ashash($file);
72
73    if (exists $$po_hash{'""'}) {
74        if ($$po_hash{'""'}->msgstr =~ /^(?:.*\\n)?Language:\s*([^\s\\]+)/im) {
75            if ($language ne $1) {
76                my $line = 3 + scalar(@{[$& =~ /(\\n)/g]});
77                print STDERR "$file:$line: Language code '$1' doesn't match '$language' from filename\n";
78            }
79        } else {
80            my $line = 2 + scalar(@{[$$po_hash{'""'}->msgstr =~ /(\\n)/g]});
81            print STDERR "$file:$line: No suitable 'Language:' field in header\n";
82        }
83    } else {
84        print STDERR "$file:1: Expected 'msgid \"\"' with header\n";
85    }
86
87    foreach my $po_entry (@{$num_list}) {
88        my $ref = $po_entry->reference;
89        (defined $ref && $ref =~ /^n:(\d+)$/m) or next;
90        my $msgno = $1;
91        my $key = $po_entry->msgid;
92        my $ent = $$po_hash{$key};
93        my $where = $file . ":" . $po_entry->loaded_line_number;
94        ${$loc{$language}}[$msgno] = $where;
95        if (defined $ent) {
96            my $msg = c_unescape($po_entry->dequote($ent->msgstr));
97            next if $msg eq '';
98            if (${$msgs{$language}}[$msgno]) {
99                print STDERR "$where: warning: already had message $msgno for language $language\n";
100            }
101            ${$msgs{$language}}[$msgno] = $msg;
102        }
103    }
104}
105
106my $lang;
107my @langs = sort grep ! /_\*$/, keys %msgs;
108
109my $num_msgs = -1;
110foreach $lang (@langs) {
111   my $aref = $msgs{$lang};
112   $num_msgs = scalar @$aref if scalar @$aref > $num_msgs;
113}
114
115foreach $lang (@langs) {
116   my $fnm = $lang;
117   $file = "$srcdir/$lang.po";
118   $fnm =~ s/(_.*)$/\U$1/;
119   open OUT, ">$fnm.msg" or die $!;
120
121   my $aref = $msgs{$lang};
122
123   my $parentaref;
124   my $mainlang = $lang;
125   $parentaref = $msgs{$mainlang} if $mainlang =~ s/_.*$//;
126
127   print OUT $magic or die $!;
128   print OUT pack("CCn", $major, $minor, $num_msgs) or die $!;
129
130   my $buff = '';
131   my $missing = 0;
132   my $retranslate = 0;
133
134   for my $n (0 .. $num_msgs - 1) {
135      my $warned = 0;
136      my $msg = $$aref[$n];
137      if (!defined $msg) {
138         $msg = $$parentaref[$n] if defined $parentaref;
139         if (!defined $msg) {
140            $msg = ${$msgs{'en'}}[$n];
141            # don't report if we have a parent (as the omission will be
142            # reported there)
143            if (defined $msg && $msg ne '' && !defined $parentaref) {
144               ++$missing;
145               $warned = 1;
146            } else {
147               $msg = '' if !defined $msg;
148            }
149         }
150      } else {
151         if ($lang ne 'en') {
152             sanity_check("Message $n in language $lang", $msg, ${$msgs{'en'}}[$n], ${$loc{$lang}}[$n]);
153         }
154      }
155      $buff .= $msg . "\0";
156   }
157
158   print OUT pack('N',length($buff)), $buff or die $!;
159   close OUT or die $!;
160
161   if ($missing || $retranslate) {
162       print STDERR "Warning: ";
163       if ($missing) {
164           print STDERR "$file: $missing missing message(s)";
165           if ($retranslate) {
166               print STDERR " and $retranslate requiring retranslation";
167           }
168       } else {
169           print STDERR "$file: $retranslate message(s) requiring retranslation";
170       }
171       print STDERR " for $lang\n";
172   }
173}
174
175sub sanity_check {
176   my ($what, $msg, $orig, $where) = @_;
177   # FIXME: Only do this if the message has the "c-format" flag.
178   # check printf-like specifiers match
179   # allow valid printf specifiers, or %<any letter> to support strftime
180   # and other printf-like formats.
181   my @pcent_m = grep /\%/, split /(%(?:[-#0 +'I]*(?:[0-9]*|\*|\*m\$)(?:\.[0-9]*)?(?:hh|ll|[hlLqjzt])?[diouxXeEfFgGaAcsCSpn]|[a-zA-Z]))/, $msg;
182   my @pcent_o = grep /\%/, split /(%(?:[-#0 +'I]*(?:[0-9]*|\*|\*m\$)(?:\.[0-9]*)?(?:hh|ll|[hlLqjzt])?[diouxXeEfFgGaAcsCSpn]|[a-zA-Z]))/, $orig;
183   while (scalar @pcent_m || scalar @pcent_o) {
184       if (!scalar @pcent_m) {
185           print STDERR "$where: warning: $what misses out \%spec $pcent_o[0]\n";
186       } elsif (!scalar @pcent_o) {
187           print STDERR "$where: warning: $what has extra \%spec $pcent_m[0]\n";
188       } elsif ($pcent_m[0] ne $pcent_o[0]) {
189           print STDERR "$where: warning: $what has \%spec $pcent_m[0] instead of $pcent_o[0]\n";
190       }
191       pop @pcent_m;
192       pop @pcent_o;
193   }
194
195   # Check for missing (or added) ellipses (...)
196   if ($msg =~ /\.\.\./ && $orig !~ /\.\.\./) {
197       print STDERR "$where: warning: $what has ellipses but original doesn't\n";
198   } elsif ($msg !~ /\.\.\./ && $orig =~ /\.\.\./) {
199       print STDERR "$where: warning: $what is missing ellipses\n";
200   }
201
202   # Check for missing (or added) menu shortcut (&)
203   if ($msg =~ /\&[A-Za-z\xc2-\xf4]/ && $orig !~ /\&[A-Za-z]/) {
204       print STDERR "$where: warning: $what has menu shortcut but original doesn't\n";
205   } elsif ($msg !~ /\&[A-Za-z\xc2-\xf4]/ && $orig =~ /\&[A-Za-z]/) {
206       print STDERR "$where: warning: $what is missing menu shortcut\n";
207   }
208
209   # Check for missing (or added) double quotes (“ and ”)
210   if (scalar($msg =~ s/(?:“|»)/$&/g) != scalar($orig =~ s/“/$&/g)) {
211       print STDERR "$where: warning: $what has different numbers of “\n";
212       print STDERR "$orig\n$msg\n\n";
213   }
214   if (scalar($msg =~ s/(?:”|«)/$&/g) != scalar($orig =~ s/”/$&/g)) {
215       print STDERR "$where: warning: $what has different numbers of ”\n";
216       print STDERR "$orig\n$msg\n\n";
217   }
218
219   # Check for missing (or added) menu accelerator "##"
220   if ($msg =~ /\#\#/ && $orig !~ /\#\#/) {
221       print STDERR "$where: warning: $what has menu accelerator but original doesn't\n";
222   } elsif ($msg !~ /\#\#/ && $orig =~ /\#\#/) {
223       print STDERR "$where: warning: $what is missing menu accelerator\n";
224   } elsif ($orig =~ /\#\#(.*)/) {
225       my $acc_o = $1;
226       my ($acc_m) = $msg =~ /\#\#(.*)/;
227       if ($acc_o ne $acc_m) {
228           print STDERR "$where: warning: $what has menu accelerator $acc_m instead of $acc_o\n";
229       }
230   }
231
232   # Check for missing (or added) menu accelerator "\t"
233   if ($msg =~ /\t/ && $orig !~ /\t/) {
234       print STDERR "$where: warning: $what has menu accelerator but original doesn't\n";
235   } elsif ($msg !~ /\t/ && $orig =~ /\t/) {
236       print STDERR "$where: warning: $what is missing menu accelerator\n";
237   } elsif ($orig =~ /\t(.*)/) {
238       my $acc_o = $1;
239       my ($acc_m) = $msg =~ /\t(.*)/;
240       if ($acc_o ne $acc_m) {
241           print STDERR "$where: warning: $what has menu accelerator $acc_m instead of $acc_o\n";
242       }
243   }
244}
245
246sub c_unescape {
247   my $str = shift @_;
248   $str =~ s/\\(x..|0|[0-7][0-7][0-7]|.)/&c_unescape_char($1)/ge;
249   return $str;
250}
251
252sub c_unescape_char {
253    my $ch = shift @_;
254    if ($ch eq '0' || $ch eq 'x00' || $ch eq '000') {
255        print STDERR "Nul byte in translation! (\\$ch)\n";
256        exit(1);
257    }
258    return $ch if $ch eq '"' || $ch eq '\\';
259    return "\n" if $ch eq "n";
260    return "\t" if $ch eq "t";
261    return "\r" if $ch eq "r";
262    return "\f" if $ch eq "f";
263    return chr(hex(substr($ch,1))) if $ch =~ /^x../;
264    return chr(oct($ch)) if $ch =~ /^[0-7][0-7][0-7]/;
265    print STDERR "Unknown C-escape in translation! (\\$ch)\n";
266    exit(1);
267}
Note: See TracBrowser for help on using the repository browser.