| #include <iostream> |
| #include <string> |
|
|
| #ifdef WITH_THREADS |
| #include <boost/thread/thread.hpp> |
| #endif |
|
|
| #include "moses/TranslationModel/CompactPT/LexicalReorderingTableCreator.h" |
|
|
| #include "util/file.hh" |
|
|
| using namespace Moses; |
|
|
| void printHelp(char **argv) |
| { |
| std::cerr << "Usage " << argv[0] << ":\n" |
| " options: \n" |
| "\t-in string -- input table file name\n" |
| "\t-out string -- prefix of binary table file\n" |
| "\t-T string -- path to temporary directory (uses /tmp by default)\n" |
| #ifdef WITH_THREADS |
| "\t-threads int|all -- number of threads used for conversion\n" |
| #endif |
| "\n advanced:\n" |
| "\t-landmark int -- use landmark phrase every 2^n phrases\n" |
| "\t-fingerprint int -- number of bits used for phrase fingerprints\n" |
| "\t-join-scores -- single set of Huffman codes for score components\n" |
| "\t-quantize int -- maximum number of scores per score component\n" |
| "\n" |
| " For more information see: http://www.statmt.org/moses/?n=Moses.AdvancedFeatures#ntoc6\n\n" |
| " If you use this please cite:\n\n" |
| " @article { junczys_pbml98_2012,\n" |
| " author = { Marcin Junczys-Dowmunt },\n" |
| " title = { Phrasal Rank-Encoding: Exploiting Phrase Redundancy and\n" |
| " Translational Relations for Phrase Table Compression },\n" |
| " journal = { The Prague Bulletin of Mathematical Linguistics },\n" |
| " volume = { 98 },\n" |
| " year = { 2012 },\n" |
| " note = { Proceedings of the MT Marathon 2012, Edinburgh },\n" |
| " }\n\n" |
| " Acknowledgments: Part of this research was carried out at and funded by\n" |
| " the World Intellectual Property Organization (WIPO) in Geneva.\n\n"; |
| } |
|
|
| int main(int argc, char** argv) |
| { |
|
|
| std::string inFilePath; |
| std::string outFilePath("out"); |
| std::string tempfilePath; |
|
|
| size_t orderBits = 10; |
| size_t fingerPrintBits = 16; |
| bool multipleScoreTrees = true; |
| size_t quantize = 0; |
|
|
| size_t threads = |
| #ifdef WITH_THREADS |
| boost::thread::hardware_concurrency() ? boost::thread::hardware_concurrency() : |
| #endif |
| 1; |
|
|
| if(1 >= argc) { |
| printHelp(argv); |
| return 1; |
| } |
| for(int i = 1; i < argc; ++i) { |
| std::string arg(argv[i]); |
| if("-in" == arg && i+1 < argc) { |
| ++i; |
| inFilePath = argv[i]; |
| } else if("-out" == arg && i+1 < argc) { |
| ++i; |
| outFilePath = argv[i]; |
| } else if("-T" == arg && i+1 < argc) { |
| ++i; |
| tempfilePath = argv[i]; |
| util::NormalizeTempPrefix(tempfilePath); |
| } else if("-landmark" == arg && i+1 < argc) { |
| ++i; |
| orderBits = atoi(argv[i]); |
| } else if("-fingerprint" == arg && i+1 < argc) { |
| ++i; |
| fingerPrintBits = atoi(argv[i]); |
| } else if("-join-scores" == arg) { |
| multipleScoreTrees = false; |
| } else if("-quantize" == arg && i+1 < argc) { |
| ++i; |
| quantize = atoi(argv[i]); |
| } else if("-threads" == arg && i+1 < argc) { |
| #ifdef WITH_THREADS |
| ++i; |
| if(std::string(argv[i]) == "all") { |
| threads = boost::thread::hardware_concurrency(); |
| if(!threads) { |
| std::cerr << "Could not determine number of hardware threads, setting to 1" << std::endl; |
| threads = 1; |
| } |
| } else |
| threads = atoi(argv[i]); |
| #else |
| std::cerr << "Thread support not compiled in" << std::endl; |
| exit(1); |
| #endif |
| } else { |
| |
| printHelp(argv); |
| return 1; |
| } |
| } |
|
|
| if(outFilePath.rfind(".minlexr") != outFilePath.size() - 8) |
| outFilePath += ".minlexr"; |
|
|
| LexicalReorderingTableCreator( |
| inFilePath, outFilePath, tempfilePath, |
| orderBits, fingerPrintBits, |
| multipleScoreTrees, quantize |
| #ifdef WITH_THREADS |
| , threads |
| #endif |
| ); |
| } |
|
|
