Upload folder using huggingface_hub

41f6dd8 verified almost 2 years ago

11 kB

// $Id$
// vim:tabstop=2
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#ifndef moses_PhraseTableCreator_h
#define moses_PhraseTableCreator_h

#include <sstream>
#include <iostream>
#include <queue>
#include <vector>
#include <set>
#include <boost/unordered_map.hpp>

#include "moses/InputFileStream.h"
#include "moses/ThreadPool.h"
#include "moses/Util.h"

#include "BlockHashIndex.h"
#include "StringVector.h"
#include "StringVectorTemp.h"
#include "CanonicalHuffman.h"

namespace Moses
{

typedef std::pair<unsigned char, unsigned char> AlignPoint;

template <typename DataType>
class Counter
{
public:
  typedef boost::unordered_map<DataType, size_t> FreqMap;
  typedef typename FreqMap::iterator iterator;
  typedef typename FreqMap::mapped_type mapped_type;
  typedef typename FreqMap::value_type value_type;

private:
#ifdef WITH_THREADS
  boost::mutex m_mutex;
#endif
  FreqMap m_freqMap;
  size_t m_maxSize;
  std::vector<DataType> m_bestVec;

  struct FreqSorter {
    bool operator()(const value_type& a, const value_type& b) const {
      if(a.second > b.second)
        return true;
      // Check impact on translation quality!
      if(a.second == b.second && a.first > b.first)
        return true;
      return false;
    }
  };

public:
  Counter() : m_maxSize(0) {}

  iterator Begin() {
    return m_freqMap.begin();
  }

  iterator End() {
    return m_freqMap.end();
  }

  void Increase(DataType data) {
#ifdef WITH_THREADS
    boost::mutex::scoped_lock lock(m_mutex);
#endif
    m_freqMap[data]++;
  }

  void IncreaseBy(DataType data, size_t num) {
#ifdef WITH_THREADS
    boost::mutex::scoped_lock lock(m_mutex);
#endif
    m_freqMap[data] += num;
  }

  mapped_type& operator[](DataType data) {
    return m_freqMap[data];
  }

  size_t Size() {
#ifdef WITH_THREADS
    boost::mutex::scoped_lock lock(m_mutex);
#endif
    return m_freqMap.size();
  }

  void Quantize(size_t maxSize) {
#ifdef WITH_THREADS
    boost::mutex::scoped_lock lock(m_mutex);
#endif
    m_maxSize = maxSize;
    std::vector<std::pair<DataType, mapped_type> > freqVec;
    freqVec.insert(freqVec.begin(), m_freqMap.begin(), m_freqMap.end());
    std::sort(freqVec.begin(), freqVec.end(), FreqSorter());

    for(size_t i = 0; i < freqVec.size() && i < m_maxSize; i++)
      m_bestVec.push_back(freqVec[i].first);

    std::sort(m_bestVec.begin(), m_bestVec.end());

    FreqMap t_freqMap;
    for(typename std::vector<std::pair<DataType, mapped_type> >::iterator it
        = freqVec.begin(); it != freqVec.end(); it++) {
      DataType closest = LowerBound(it->first);
      t_freqMap[closest] += it->second;
    }

    m_freqMap.swap(t_freqMap);
  }

  void Clear() {
#ifdef WITH_THREADS
    boost::mutex::scoped_lock lock(m_mutex);
#endif
    m_freqMap.clear();
  }

  DataType LowerBound(DataType data) {
    if(m_maxSize == 0 || m_bestVec.size() == 0)
      return data;
    else {
      typename std::vector<DataType>::iterator it
      = std::lower_bound(m_bestVec.begin(), m_bestVec.end(), data);
      if(it != m_bestVec.end())
        return *it;
      else
        return m_bestVec.back();
    }
  }
};

class PackedItem
{
private:
  long m_line;
  std::string m_sourcePhrase;
  std::string m_packedTargetPhrase;
  size_t m_rank;
  float m_score;

public:
  PackedItem(long line, std::string sourcePhrase,
             std::string packedTargetPhrase, size_t rank,
             float m_score = 0);

  long GetLine() const;
  const std::string& GetSrc() const;
  const std::string& GetTrg() const;
  size_t GetRank() const;
  float GetScore() const;
};

bool operator<(const PackedItem &pi1, const PackedItem &pi2);

class PhraseTableCreator
{
public:
  enum Coding { None, REnc, PREnc };

private:
  std::string m_inPath;
  std::string m_outPath;
  std::string m_tempfilePath;

  std::FILE* m_outFile;

  size_t m_numScoreComponent;
  size_t m_sortScoreIndex;
  size_t m_warnMe;

  Coding m_coding;
  size_t m_orderBits;
  size_t m_fingerPrintBits;
  bool m_useAlignmentInfo;
  bool m_multipleScoreTrees;
  size_t m_quantize;
  size_t m_maxRank;

  static std::string m_phraseStopSymbol;
  static std::string m_separator;

#ifdef WITH_THREADS
  size_t m_threads;
  boost::mutex m_mutex;
#endif

  BlockHashIndex m_srcHash;
  BlockHashIndex m_rnkHash;

  size_t m_maxPhraseLength;

  std::vector<unsigned> m_ranks;

  typedef std::pair<unsigned, unsigned> SrcTrg;
  typedef std::pair<std::string, std::string> SrcTrgString;
  typedef std::pair<SrcTrgString, float> SrcTrgProb;

  struct SrcTrgProbSorter {
    bool operator()(const SrcTrgProb& a, const SrcTrgProb& b) const {
      if(a.first.first < b.first.first)
        return true;

      if(a.first.first == b.first.first && a.second > b.second)
        return true;

      if(a.first.first == b.first.first
          && a.second == b.second
          && a.first.second < b.first.second)
        return true;

      return false;
    }
  };

  std::vector<size_t> m_lexicalTableIndex;
  std::vector<SrcTrg> m_lexicalTable;

  StringVectorTemp<unsigned char, unsigned long, MmapAllocator>*
  m_encodedTargetPhrases;

  StringVector<unsigned char, unsigned long, MmapAllocator>*
  m_compressedTargetPhrases;

  boost::unordered_map<std::string, unsigned> m_targetSymbolsMap;
  boost::unordered_map<std::string, unsigned> m_sourceSymbolsMap;

  typedef Counter<unsigned> SymbolCounter;
  typedef Counter<float> ScoreCounter;
  typedef Counter<AlignPoint> AlignCounter;

  typedef CanonicalHuffman<unsigned> SymbolTree;
  typedef CanonicalHuffman<float> ScoreTree;
  typedef CanonicalHuffman<AlignPoint> AlignTree;

  SymbolCounter m_symbolCounter;
  SymbolTree* m_symbolTree;

  AlignCounter m_alignCounter;
  AlignTree* m_alignTree;

  std::vector<ScoreCounter*> m_scoreCounters;
  std::vector<ScoreTree*> m_scoreTrees;

  std::priority_queue<PackedItem> m_queue;
  long m_lastFlushedLine;
  long m_lastFlushedSourceNum;
  std::string m_lastFlushedSourcePhrase;
  std::vector<std::string> m_lastSourceRange;
  std::priority_queue<std::pair<float, size_t> > m_rankQueue;
  std::vector<std::string> m_lastCollection;

  void Save();
  void PrintInfo();

  void AddSourceSymbolId(std::string& symbol);
  unsigned GetSourceSymbolId(std::string& symbol);

  void AddTargetSymbolId(std::string& symbol);
  unsigned GetTargetSymbolId(std::string& symbol);
  unsigned GetOrAddTargetSymbolId(std::string& symbol);

  unsigned GetRank(unsigned srcIdx, unsigned trgIdx);

  unsigned EncodeREncSymbol1(unsigned symbol);
  unsigned EncodeREncSymbol2(unsigned position, unsigned rank);
  unsigned EncodeREncSymbol3(unsigned rank);

  unsigned EncodePREncSymbol1(unsigned symbol);
  unsigned EncodePREncSymbol2(int lOff, int rOff, unsigned rank);

  void EncodeTargetPhraseNone(std::vector<std::string>& t,
                              std::ostream& os);

  void EncodeTargetPhraseREnc(std::vector<std::string>& s,
                              std::vector<std::string>& t,
                              std::set<AlignPoint>& a,
                              std::ostream& os);

  void EncodeTargetPhrasePREnc(std::vector<std::string>& s,
                               std::vector<std::string>& t,
                               std::set<AlignPoint>& a, size_t ownRank,
                               std::ostream& os);

  void EncodeScores(std::vector<float>& scores, std::ostream& os);
  void EncodeAlignment(std::set<AlignPoint>& alignment, std::ostream& os);

  std::string MakeSourceKey(std::string&);
  std::string MakeSourceTargetKey(std::string&, std::string&);

  void LoadLexicalTable(std::string filePath);

  void CreateRankHash();
  void EncodeTargetPhrases();
  void CalcHuffmanCodes();
  void CompressTargetPhrases();

  void AddRankedLine(PackedItem& pi);
  void FlushRankedQueue(bool force = false);

  std::string EncodeLine(std::vector<std::string>& tokens, size_t ownRank);
  void AddEncodedLine(PackedItem& pi);
  void FlushEncodedQueue(bool force = false);

  std::string CompressEncodedCollection(std::string encodedCollection);
  void AddCompressedCollection(PackedItem& pi);
  void FlushCompressedQueue(bool force = false);

public:

  PhraseTableCreator(std::string inPath,
                     std::string outPath,
                     std::string tempfilePath,
                     size_t numScoreComponent = 5,
                     size_t sortScoreIndex = 2,
                     Coding coding = PREnc,
                     size_t orderBits = 10,
                     size_t fingerPrintBits = 16,
                     bool useAlignmentInfo = false,
                     bool multipleScoreTrees = true,
                     size_t quantize = 0,
                     size_t maxRank = 100,
                     bool warnMe = true
#ifdef WITH_THREADS
                                   , size_t threads = 2
#endif
                    );

  ~PhraseTableCreator();

  friend class RankingTask;
  friend class EncodingTask;
  friend class CompressionTask;
};

class RankingTask
{
private:
#ifdef WITH_THREADS
  static boost::mutex m_mutex;
  static boost::mutex m_fileMutex;
#endif
  static size_t m_lineNum;
  InputFileStream& m_inFile;
  PhraseTableCreator& m_creator;

public:
  RankingTask(InputFileStream& inFile, PhraseTableCreator& creator);
  void operator()();
};

class EncodingTask
{
private:
#ifdef WITH_THREADS
  static boost::mutex m_mutex;
  static boost::mutex m_fileMutex;
#endif
  static size_t m_lineNum;
  static size_t m_sourcePhraseNum;
  static std::string m_lastSourcePhrase;

  InputFileStream& m_inFile;
  PhraseTableCreator& m_creator;

public:
  EncodingTask(InputFileStream& inFile, PhraseTableCreator& creator);
  void operator()();
};

class CompressionTask
{
private:
#ifdef WITH_THREADS
  static boost::mutex m_mutex;
#endif
  static size_t m_collectionNum;
  StringVectorTemp<unsigned char, unsigned long, MmapAllocator>&
  m_encodedCollections;
  PhraseTableCreator& m_creator;

public:
  CompressionTask(StringVectorTemp<unsigned char, unsigned long, MmapAllocator>&
                  encodedCollections, PhraseTableCreator& creator);
  void operator()();
};

}

#endif

	// $Id$
	// vim:tabstop=2
	/***********************************************************************
	Moses - factored phrase-based language decoder
	Copyright (C) 2006 University of Edinburgh

	This library is free software; you can redistribute it and/or
	modify it under the terms of the GNU Lesser General Public
	License as published by the Free Software Foundation; either
	version 2.1 of the License, or (at your option) any later version.

	This library is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	Lesser General Public License for more details.

	You should have received a copy of the GNU Lesser General Public
	License along with this library; if not, write to the Free Software
	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	***********************************************************************/

	#ifndef moses_PhraseTableCreator_h
	#define moses_PhraseTableCreator_h

	#include <sstream>
	#include <iostream>
	#include <queue>
	#include <vector>
	#include <set>
	#include <boost/unordered_map.hpp>

	#include "moses/InputFileStream.h"
	#include "moses/ThreadPool.h"
	#include "moses/Util.h"

	#include "BlockHashIndex.h"
	#include "StringVector.h"
	#include "StringVectorTemp.h"
	#include "CanonicalHuffman.h"

	namespace Moses
	{

	typedef std::pair<unsigned char, unsigned char> AlignPoint;

	template <typename DataType>
	class Counter
	{
	public:
	typedef boost::unordered_map<DataType, size_t> FreqMap;
	typedef typename FreqMap::iterator iterator;
	typedef typename FreqMap::mapped_type mapped_type;
	typedef typename FreqMap::value_type value_type;

	private:
	#ifdef WITH_THREADS
	boost::mutex m_mutex;
	#endif
	FreqMap m_freqMap;
	size_t m_maxSize;
	std::vector<DataType> m_bestVec;

	struct FreqSorter {
	bool operator()(const value_type& a, const value_type& b) const {
	if(a.second > b.second)
	return true;
	// Check impact on translation quality!
	if(a.second == b.second && a.first > b.first)
	return true;
	return false;
	}
	};

	public:
	Counter() : m_maxSize(0) {}

	iterator Begin() {
	return m_freqMap.begin();
	}

	iterator End() {
	return m_freqMap.end();
	}

	void Increase(DataType data) {
	#ifdef WITH_THREADS
	boost::mutex::scoped_lock lock(m_mutex);
	#endif
	m_freqMap[data]++;
	}

	void IncreaseBy(DataType data, size_t num) {
	#ifdef WITH_THREADS
	boost::mutex::scoped_lock lock(m_mutex);
	#endif
	m_freqMap[data] += num;
	}

	mapped_type& operator[](DataType data) {
	return m_freqMap[data];
	}

	size_t Size() {
	#ifdef WITH_THREADS
	boost::mutex::scoped_lock lock(m_mutex);
	#endif
	return m_freqMap.size();
	}

	void Quantize(size_t maxSize) {
	#ifdef WITH_THREADS
	boost::mutex::scoped_lock lock(m_mutex);
	#endif
	m_maxSize = maxSize;
	std::vector<std::pair<DataType, mapped_type> > freqVec;
	freqVec.insert(freqVec.begin(), m_freqMap.begin(), m_freqMap.end());
	std::sort(freqVec.begin(), freqVec.end(), FreqSorter());

	for(size_t i = 0; i < freqVec.size() && i < m_maxSize; i++)
	m_bestVec.push_back(freqVec[i].first);

	std::sort(m_bestVec.begin(), m_bestVec.end());

	FreqMap t_freqMap;
	for(typename std::vector<std::pair<DataType, mapped_type> >::iterator it
	= freqVec.begin(); it != freqVec.end(); it++) {
	DataType closest = LowerBound(it->first);
	t_freqMap[closest] += it->second;
	}

	m_freqMap.swap(t_freqMap);
	}

	void Clear() {
	#ifdef WITH_THREADS
	boost::mutex::scoped_lock lock(m_mutex);
	#endif
	m_freqMap.clear();
	}

	DataType LowerBound(DataType data) {
	if(m_maxSize == 0 \|\| m_bestVec.size() == 0)
	return data;
	else {
	typename std::vector<DataType>::iterator it
	= std::lower_bound(m_bestVec.begin(), m_bestVec.end(), data);
	if(it != m_bestVec.end())
	return *it;
	else
	return m_bestVec.back();
	}
	}
	};

	class PackedItem
	{
	private:
	long m_line;
	std::string m_sourcePhrase;
	std::string m_packedTargetPhrase;
	size_t m_rank;
	float m_score;

	public:
	PackedItem(long line, std::string sourcePhrase,
	std::string packedTargetPhrase, size_t rank,
	float m_score = 0);

	long GetLine() const;
	const std::string& GetSrc() const;
	const std::string& GetTrg() const;
	size_t GetRank() const;
	float GetScore() const;
	};

	bool operator<(const PackedItem &pi1, const PackedItem &pi2);

	class PhraseTableCreator
	{
	public:
	enum Coding { None, REnc, PREnc };

	private:
	std::string m_inPath;
	std::string m_outPath;
	std::string m_tempfilePath;

	std::FILE* m_outFile;

	size_t m_numScoreComponent;
	size_t m_sortScoreIndex;
	size_t m_warnMe;

	Coding m_coding;
	size_t m_orderBits;
	size_t m_fingerPrintBits;
	bool m_useAlignmentInfo;
	bool m_multipleScoreTrees;
	size_t m_quantize;
	size_t m_maxRank;

	static std::string m_phraseStopSymbol;
	static std::string m_separator;

	#ifdef WITH_THREADS
	size_t m_threads;
	boost::mutex m_mutex;
	#endif

	BlockHashIndex m_srcHash;
	BlockHashIndex m_rnkHash;

	size_t m_maxPhraseLength;

	std::vector<unsigned> m_ranks;

	typedef std::pair<unsigned, unsigned> SrcTrg;
	typedef std::pair<std::string, std::string> SrcTrgString;
	typedef std::pair<SrcTrgString, float> SrcTrgProb;

	struct SrcTrgProbSorter {
	bool operator()(const SrcTrgProb& a, const SrcTrgProb& b) const {
	if(a.first.first < b.first.first)
	return true;

	if(a.first.first == b.first.first && a.second > b.second)
	return true;

	if(a.first.first == b.first.first
	&& a.second == b.second
	&& a.first.second < b.first.second)
	return true;

	return false;
	}
	};

	std::vector<size_t> m_lexicalTableIndex;
	std::vector<SrcTrg> m_lexicalTable;

	StringVectorTemp<unsigned char, unsigned long, MmapAllocator>*
	m_encodedTargetPhrases;

	StringVector<unsigned char, unsigned long, MmapAllocator>*
	m_compressedTargetPhrases;

	boost::unordered_map<std::string, unsigned> m_targetSymbolsMap;
	boost::unordered_map<std::string, unsigned> m_sourceSymbolsMap;

	typedef Counter<unsigned> SymbolCounter;
	typedef Counter<float> ScoreCounter;
	typedef Counter<AlignPoint> AlignCounter;

	typedef CanonicalHuffman<unsigned> SymbolTree;
	typedef CanonicalHuffman<float> ScoreTree;
	typedef CanonicalHuffman<AlignPoint> AlignTree;

	SymbolCounter m_symbolCounter;
	SymbolTree* m_symbolTree;

	AlignCounter m_alignCounter;
	AlignTree* m_alignTree;

	std::vector<ScoreCounter*> m_scoreCounters;
	std::vector<ScoreTree*> m_scoreTrees;

	std::priority_queue<PackedItem> m_queue;
	long m_lastFlushedLine;
	long m_lastFlushedSourceNum;
	std::string m_lastFlushedSourcePhrase;
	std::vector<std::string> m_lastSourceRange;
	std::priority_queue<std::pair<float, size_t> > m_rankQueue;
	std::vector<std::string> m_lastCollection;

	void Save();
	void PrintInfo();

	void AddSourceSymbolId(std::string& symbol);
	unsigned GetSourceSymbolId(std::string& symbol);

	void AddTargetSymbolId(std::string& symbol);
	unsigned GetTargetSymbolId(std::string& symbol);
	unsigned GetOrAddTargetSymbolId(std::string& symbol);

	unsigned GetRank(unsigned srcIdx, unsigned trgIdx);

	unsigned EncodeREncSymbol1(unsigned symbol);
	unsigned EncodeREncSymbol2(unsigned position, unsigned rank);
	unsigned EncodeREncSymbol3(unsigned rank);

	unsigned EncodePREncSymbol1(unsigned symbol);
	unsigned EncodePREncSymbol2(int lOff, int rOff, unsigned rank);

	void EncodeTargetPhraseNone(std::vector<std::string>& t,
	std::ostream& os);

	void EncodeTargetPhraseREnc(std::vector<std::string>& s,
	std::vector<std::string>& t,
	std::set<AlignPoint>& a,
	std::ostream& os);

	void EncodeTargetPhrasePREnc(std::vector<std::string>& s,
	std::vector<std::string>& t,
	std::set<AlignPoint>& a, size_t ownRank,
	std::ostream& os);

	void EncodeScores(std::vector<float>& scores, std::ostream& os);
	void EncodeAlignment(std::set<AlignPoint>& alignment, std::ostream& os);

	std::string MakeSourceKey(std::string&);
	std::string MakeSourceTargetKey(std::string&, std::string&);

	void LoadLexicalTable(std::string filePath);

	void CreateRankHash();
	void EncodeTargetPhrases();
	void CalcHuffmanCodes();
	void CompressTargetPhrases();

	void AddRankedLine(PackedItem& pi);
	void FlushRankedQueue(bool force = false);

	std::string EncodeLine(std::vector<std::string>& tokens, size_t ownRank);
	void AddEncodedLine(PackedItem& pi);
	void FlushEncodedQueue(bool force = false);

	std::string CompressEncodedCollection(std::string encodedCollection);
	void AddCompressedCollection(PackedItem& pi);
	void FlushCompressedQueue(bool force = false);

	public:

	PhraseTableCreator(std::string inPath,
	std::string outPath,
	std::string tempfilePath,
	size_t numScoreComponent = 5,
	size_t sortScoreIndex = 2,
	Coding coding = PREnc,
	size_t orderBits = 10,
	size_t fingerPrintBits = 16,
	bool useAlignmentInfo = false,
	bool multipleScoreTrees = true,
	size_t quantize = 0,
	size_t maxRank = 100,
	bool warnMe = true
	#ifdef WITH_THREADS
	, size_t threads = 2
	#endif
	);

	~PhraseTableCreator();

	friend class RankingTask;
	friend class EncodingTask;
	friend class CompressionTask;
	};

	class RankingTask
	{
	private:
	#ifdef WITH_THREADS
	static boost::mutex m_mutex;
	static boost::mutex m_fileMutex;
	#endif
	static size_t m_lineNum;
	InputFileStream& m_inFile;
	PhraseTableCreator& m_creator;

	public:
	RankingTask(InputFileStream& inFile, PhraseTableCreator& creator);
	void operator()();
	};

	class EncodingTask
	{
	private:
	#ifdef WITH_THREADS
	static boost::mutex m_mutex;
	static boost::mutex m_fileMutex;
	#endif
	static size_t m_lineNum;
	static size_t m_sourcePhraseNum;
	static std::string m_lastSourcePhrase;

	InputFileStream& m_inFile;
	PhraseTableCreator& m_creator;

	public:
	EncodingTask(InputFileStream& inFile, PhraseTableCreator& creator);
	void operator()();
	};

	class CompressionTask
	{
	private:
	#ifdef WITH_THREADS
	static boost::mutex m_mutex;
	#endif
	static size_t m_collectionNum;
	StringVectorTemp<unsigned char, unsigned long, MmapAllocator>&
	m_encodedCollections;
	PhraseTableCreator& m_creator;

	public:
	CompressionTask(StringVectorTemp<unsigned char, unsigned long, MmapAllocator>&
	encodedCollections, PhraseTableCreator& creator);
	void operator()();
	};

	}

	#endif