Hugging Face's logo

Quagmire1
/
wiki-cased

Model card Files
xet
 Community
wiki-cased / tools /mosesdecoder /scripts /generic /giza-parallel.perl
Quagmire1's picture
Quagmire1
Upload folder using huggingface_hub
41f6dd8 verified
Raw History Blame Contribute Delete
3.45 kB
#!/usr/bin/env perl
#
# This file is part of moses. Its use is licensed under the GNU Lesser General
# Public License version 2.1 or, at your option, any later version.
# example
# ~/giza-parallel.perl 10 split ~/workspace/sourceforge/trunk/scripts/training/train-model.perl ar en train align
use warnings;
use strict;
use File::Basename;
sub NumStr($);
print "Started ".localtime() ."\n";
my $numParallel = $ARGV[0];
my $splitCmd = $ARGV[1];
my $trainCmd = $ARGV[2];
my $inputExt = $ARGV[3];
my $outputExt = $ARGV[4];
my $corpus = $ARGV[5];
my $align = $ARGV[6];
my $TMPDIR=dirname($align) ."/tmp.$$";
mkdir $TMPDIR;
my $scriptDir=dirname($trainCmd) ."/..";
# split corpus file
my $totalLines = int(`wc -l $corpus.$inputExt`);
my $linesPerSplit = int($totalLines / $numParallel) + 1;
my $cmd = "$splitCmd -d -l $linesPerSplit -a 5 $corpus.$inputExt $TMPDIR/source.";
`$cmd`;
$cmd = "$splitCmd -d -l $linesPerSplit -a 5 $corpus.$outputExt $TMPDIR/target.";
`$cmd`;
for (my $i = 0; $i < $numParallel; ++$i)
{
my $numStr = NumStr($i);
rename("$TMPDIR/source.$numStr", "$TMPDIR/$numStr.source");
rename("$TMPDIR/target.$numStr", "$TMPDIR/$numStr.target");
}
#fork & run giza & friends
my $isParent = 1;
my @childs;
for (my $i = 0; $i < $numParallel; ++$i)
{
my $pid = fork();
if ($pid == 0)
{ # child
$isParent = 0;
my $numStr = NumStr($i);
my $cmd = "$trainCmd -dont-zip -last-step 1 -scripts-root-dir $scriptDir -f source -e target -alignment grow-diag-final-and -parts 3 -reordering msd-bidirectional-fe -corpus $TMPDIR/$numStr -corpus-dir $TMPDIR/prepared.$numStr \n";
print $cmd;
`$cmd`;
$cmd = "$trainCmd -dont-zip -first-step 2 -last-step 2 -scripts-root-dir $scriptDir -f source -e target -alignment grow-diag-final-and -parts 3 -reordering msd-bidirectional-fe -corpus-dir $TMPDIR/prepared.$numStr -giza-e2f $TMPDIR/giza.$numStr -direction 2 \n";
print $cmd;
`$cmd`;
$cmd = "$trainCmd -dont-zip -first-step 2 -last-step 2 -scripts-root-dir $scriptDir -f source -e target -alignment grow-diag-final-and -parts 3 -reordering msd-bidirectional-fe -corpus-dir $TMPDIR/prepared.$numStr -giza-f2e $TMPDIR/giza-inverse.$numStr -direction 1 \n";
print $cmd;
`$cmd`;
$cmd = "$trainCmd -dont-zip -first-step 3 -last-step 3 -scripts-root-dir $scriptDir -f source -e target -alignment grow-diag-final-and -parts 3 -reordering msd-bidirectional-fe -giza-e2f $TMPDIR/giza.$numStr -giza-f2e $TMPDIR/giza-inverse.$numStr -alignment-file $TMPDIR/aligned.$numStr -alignment grow-diag-final-and \n";
print $cmd;
`$cmd`;
exit();
}
else
{ # parent
push(@childs, $pid);
}
}
# wait for everything is finished
if ($isParent)
{
foreach (@childs) {
waitpid($_, 0);
}
}
else
{
die "shouldn't be here";
}
# cat all aligned files together. Voila
my $cmd = "cat ";
for (my $i = 0; $i < $numParallel; ++$i)
{
my $numStr = NumStr($i);
$cmd .= "$TMPDIR/aligned.$numStr.grow-diag-final-and ";
}
$cmd .= " > $align \n";
print $cmd;
`$cmd`;
sub NumStr($)
{
my $i = shift;
my $numStr;
if ($i < 10) {
$numStr = "000000$i";
}
elsif ($i < 100) {
$numStr = "00000$i";
}
elsif ($i < 1000) {
$numStr = "0000$i";
}
elsif ($i < 10000) {
$numStr = "000$i";
}
elsif ($i < 100000) {
$numStr = "00$i";
}
elsif ($i < 1000000) {
$numStr = "0$i";
}
else {
$numStr = $i;
}
return $numStr;
}