File size: 3,546 Bytes
cd6775d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | # Copyright (c) 2020-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
#
# Usage: ./get-data-umt.sh --src mh --tgt sh --reload_codes dumped/xlm_en/codes_en --reload_vocab dumped/xlm_en/vocab_en --data_folder all
# This script will successively:
# 1) download Moses scripts, download and compile fastBPE
# 2) download, extract, tokenize, apply BPE to monolingual and parallel test data
# 3) binarize all datasets
set -e
#
# Read arguments
#
POSITIONAL=()
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
--src)
SRC="$2"; shift 2;;
--tgt)
TGT="$2"; shift 2;;
--reload_codes)
RELOAD_CODES="$2"; shift 2;;
--reload_vocab)
RELOAD_VOCAB="$2"; shift 2;;
--data_folder)
DATA_FOLDER="$2"; shift 2;;
--split)
split="$2"; shift 2;;
*)
POSITIONAL+=("$1")
shift
;;
esac
done
set -- "${POSITIONAL[@]}"
#
# Check parameters
#
if [ "$SRC" == "" ]; then echo "--src not provided"; exit; fi
if [ "$TGT" == "" ]; then echo "--tgt not provided"; exit; fi
if [ "$SRC" != "mh" -a "$SRC" != "sh" ]; then echo "unknown source language"; exit; fi
if [ "$TGT" != "mh" -a "$TGT" != "sh" ]; then echo "unknown target language"; exit; fi
if [ "$SRC" == "$TGT" ]; then echo "source and target cannot be identical"; exit; fi
if [ "$SRC" \> "$TGT" ]; then echo "please ensure SRC < TGT"; exit; fi
if [ "$RELOAD_CODES" != "" ] && [ ! -f "$RELOAD_CODES" ]; then echo "cannot locate BPE codes"; exit; fi
if [ "$RELOAD_VOCAB" != "" ] && [ ! -f "$RELOAD_VOCAB" ]; then echo "cannot locate vocabulary"; exit; fi
if [ "$RELOAD_CODES" == "" -a "$RELOAD_VOCAB" != "" -o "$RELOAD_CODES" != "" -a "$RELOAD_VOCAB" == "" ]; then echo "BPE codes should be provided if and only if vocabulary is also provided"; exit; fi
#
# Initialize tools and data paths
#
# main paths
MAIN_PATH=$PWD
TOOLS_PATH=$PWD/tools
TOKENIZE=$TOOLS_PATH/tokenize.sh
LOWER_REMOVE_ACCENT=$TOOLS_PATH/lowercase_and_remove_accent.py
DATA_PATH=$PWD/data/umt/$DATA_FOLDER
PROC_PATH=$DATA_PATH/processed
# create paths
mkdir -p $TOOLS_PATH
mkdir -p $PROC_PATH
# fastBPE
FASTBPE=$TOOLS_PATH/fastBPE/fast
# BPE / vocab files
BPE_CODES=$PROC_PATH/codes
FULL_VOCAB=$PROC_PATH/vocab.$SRC-$TGT
# install tools
./install-tools.sh
# reload BPE codes
echo "Reloading BPE codes from $RELOAD_CODES ..."
cp $RELOAD_CODES $BPE_CODES
# reload full vocabulary
echo "Reloading vocabulary from $RELOAD_VOCAB ..."
cp $RELOAD_VOCAB $FULL_VOCAB
# preprocess
lg=$SRC
RAW=$DATA_PATH/$split.$lg
TOK=$RAW.tok
BPE=$PROC_PATH/$split.$lg
echo "Preprocessing $split.$lg..."
if ! [[ -f "$TOK" ]]; then
echo "Tokenizing $RAW..."
eval "cat $RAW | $TOKENIZE en | python $LOWER_REMOVE_ACCENT > $TOK"
fi
if ! [[ -f "$BPE" ]]; then
echo "Applying BPE codes to $split.$lg..."
$FASTBPE applybpe $BPE $TOK $BPE_CODES $FULL_VOCAB
fi
if ! [[ -f "$BPE.pth" ]]; then
echo "Binarizing $split.$lg..."
$MAIN_PATH/preprocess.py $FULL_VOCAB $BPE
fi
#
# Link parallel validation and test data to monolingual data
#
#ln -sf $PROC_PATH/$split.$SRC.pth $PROC_PATH/$split.$SRC-$TGT.$SRC.pth
#ln -sf $PROC_PATH/$split.$TGT.pth $PROC_PATH/$split.$SRC-$TGT.$TGT.pth
#for lg in $SRC $TGT; do
# BPE=$PROC_PATH/test.$lg
# NUM_VALID=`wc -l < $PROC_PATH/valid.$lg`
# head -$NUM_VALID $PROC_PATH/train.$lg > $BPE
# $MAIN_PATH/preprocess.py $FULL_VOCAB $BPE
#done
#
# Summary
#
echo ""
echo "===== Data summary"
echo "Processed $split data:"
echo " $SRC: $PROC_PATH/$split.$SRC.pth"
echo ""
|