Add files using upload-large-folder tool

cd33cd3 verified over 1 year ago

5.18 kB

/*
 * AlignedSentence.cpp
 *
 *  Created on: 18 Feb 2014
 *      Author: s0565741
 */

#include <sstream>
#include "moses/Util.h"
#include "AlignedSentence.h"
#include "Parameter.h"

using namespace std;


/////////////////////////////////////////////////////////////////////////////////
AlignedSentence::AlignedSentence(int lineNum,
                                 const std::string &source,
                                 const std::string &target,
                                 const std::string &alignment)
  :m_lineNum(lineNum)
{
  PopulateWordVec(m_source, source);
  PopulateWordVec(m_target, target);
  PopulateAlignment(alignment);
}

AlignedSentence::~AlignedSentence()
{
  Moses::RemoveAllInColl(m_source);
  Moses::RemoveAllInColl(m_target);
}

void AlignedSentence::PopulateWordVec(Phrase &vec, const std::string &line)
{
  std::vector<string> toks;
  Moses::Tokenize(toks, line);

  vec.resize(toks.size());
  for (size_t i = 0; i < vec.size(); ++i) {
    const string &tok = toks[i];
    Word *word = new Word(i, tok);
    vec[i] = word;
  }
}

void AlignedSentence::PopulateAlignment(const std::string &line)
{
  vector<string> alignStr;
  Moses::Tokenize(alignStr, line);

  for (size_t i = 0; i < alignStr.size(); ++i) {
    vector<int> alignPair;
    Moses::Tokenize(alignPair, alignStr[i], "-");
    assert(alignPair.size() == 2);

    int sourcePos = alignPair[0];
    int targetPos = alignPair[1];

    if (sourcePos >= m_source.size()) {
      cerr << "ERROR1:AlignedSentence=" << Debug() << endl;
      cerr << "m_source=" << m_source.size() << endl;
      abort();
    }
    assert(sourcePos < m_source.size());
    assert(targetPos < m_target.size());
    Word *sourceWord = m_source[sourcePos];
    Word *targetWord = m_target[targetPos];

    sourceWord->AddAlignment(targetWord);
    targetWord->AddAlignment(sourceWord);
  }
}

std::string AlignedSentence::Debug() const
{
  stringstream out;
  out << "m_lineNum:";
  out << m_lineNum;
  out << endl;

  out << "m_source:";
  out << m_source.Debug();
  out << endl;

  out << "m_target:";
  out << m_target.Debug();
  out << endl;

  out << "consistent phrases:" << endl;
  out << m_consistentPhrases.Debug();
  out << endl;

  return out.str();
}

std::vector<int> AlignedSentence::GetSourceAlignmentCount() const
{
  vector<int> ret(m_source.size());

  for (size_t i = 0; i < m_source.size(); ++i) {
    const Word &word = *m_source[i];
    ret[i] = word.GetAlignmentIndex().size();
  }
  return ret;
}

void AlignedSentence::Create(const Parameter &params)
{
  CreateConsistentPhrases(params);
  m_consistentPhrases.AddHieroNonTerms(params);
}

void AlignedSentence::CreateConsistentPhrases(const Parameter &params)
{
  int countT = m_target.size();
  int countS = m_source.size();

  m_consistentPhrases.Initialize(countS);

  // check alignments for target phrase startT...endT
  for(int lengthT=1;
      lengthT <= params.maxSpan && lengthT <= countT;
      lengthT++) {
    for(int startT=0; startT < countT-(lengthT-1); startT++) {

      // that's nice to have
      int endT = startT + lengthT - 1;

      // find find aligned source words
      // first: find minimum and maximum source word
      int minS = 9999;
      int maxS = -1;
      vector< int > usedS = GetSourceAlignmentCount();
      for(int ti=startT; ti<=endT; ti++) {
        const Word &word = *m_target[ti];
        const std::set<int> &alignment = word.GetAlignmentIndex();

        std::set<int>::const_iterator iterAlign;
        for(iterAlign = alignment.begin(); iterAlign != alignment.end(); ++iterAlign) {
          int si = *iterAlign;
          if (si<minS) {
            minS = si;
          }
          if (si>maxS) {
            maxS = si;
          }
          usedS[ si ]--;
        }
      }

      // unaligned phrases are not allowed
      if( maxS == -1 )
        continue;

      // source phrase has to be within limits
      size_t width = maxS - minS + 1;

      if( width < params.minSpan )
        continue;

      if( width > params.maxSpan )
        continue;

      // check if source words are aligned to out of bound target words
      bool out_of_bounds = false;
      for(int si=minS; si<=maxS && !out_of_bounds; si++)
        if (usedS[si]>0) {
          out_of_bounds = true;
        }

      // if out of bound, you gotta go
      if (out_of_bounds)
        continue;

      // done with all the checks, lets go over all consistent phrase pairs
      // start point of source phrase may retreat over unaligned
      for(int startS=minS;
          (startS>=0 &&
           startS>maxS - params.maxSpan && // within length limit
           (startS==minS || m_source[startS]->GetAlignment().size()==0)); // unaligned
          startS--) {
        // end point of source phrase may advance over unaligned
        for(int endS=maxS;
            (endS<countS && endS<startS + params.maxSpan && // within length limit
             (endS==maxS || m_source[endS]->GetAlignment().size()==0)); // unaligned
            endS++) {

          // take note that this is a valid phrase alignment
          m_consistentPhrases.Add(startS, endS, startT, endT, params);
        }
      }
    }
  }
}

	/*
	* AlignedSentence.cpp
	*
	* Created on: 18 Feb 2014
	* Author: s0565741
	*/

	#include <sstream>
	#include "moses/Util.h"
	#include "AlignedSentence.h"
	#include "Parameter.h"

	using namespace std;


	/////////////////////////////////////////////////////////////////////////////////
	AlignedSentence::AlignedSentence(int lineNum,
	const std::string &source,
	const std::string &target,
	const std::string &alignment)
	:m_lineNum(lineNum)
	{
	PopulateWordVec(m_source, source);
	PopulateWordVec(m_target, target);
	PopulateAlignment(alignment);
	}

	AlignedSentence::~AlignedSentence()
	{
	Moses::RemoveAllInColl(m_source);
	Moses::RemoveAllInColl(m_target);
	}

	void AlignedSentence::PopulateWordVec(Phrase &vec, const std::string &line)
	{
	std::vector<string> toks;
	Moses::Tokenize(toks, line);

	vec.resize(toks.size());
	for (size_t i = 0; i < vec.size(); ++i) {
	const string &tok = toks[i];
	Word *word = new Word(i, tok);
	vec[i] = word;
	}
	}

	void AlignedSentence::PopulateAlignment(const std::string &line)
	{
	vector<string> alignStr;
	Moses::Tokenize(alignStr, line);

	for (size_t i = 0; i < alignStr.size(); ++i) {
	vector<int> alignPair;
	Moses::Tokenize(alignPair, alignStr[i], "-");
	assert(alignPair.size() == 2);

	int sourcePos = alignPair[0];
	int targetPos = alignPair[1];

	if (sourcePos >= m_source.size()) {
	cerr << "ERROR1:AlignedSentence=" << Debug() << endl;
	cerr << "m_source=" << m_source.size() << endl;
	abort();
	}
	assert(sourcePos < m_source.size());
	assert(targetPos < m_target.size());
	Word *sourceWord = m_source[sourcePos];
	Word *targetWord = m_target[targetPos];

	sourceWord->AddAlignment(targetWord);
	targetWord->AddAlignment(sourceWord);
	}
	}

	std::string AlignedSentence::Debug() const
	{
	stringstream out;
	out << "m_lineNum:";
	out << m_lineNum;
	out << endl;

	out << "m_source:";
	out << m_source.Debug();
	out << endl;

	out << "m_target:";
	out << m_target.Debug();
	out << endl;

	out << "consistent phrases:" << endl;
	out << m_consistentPhrases.Debug();
	out << endl;

	return out.str();
	}

	std::vector<int> AlignedSentence::GetSourceAlignmentCount() const
	{
	vector<int> ret(m_source.size());

	for (size_t i = 0; i < m_source.size(); ++i) {
	const Word &word = *m_source[i];
	ret[i] = word.GetAlignmentIndex().size();
	}
	return ret;
	}

	void AlignedSentence::Create(const Parameter &params)
	{
	CreateConsistentPhrases(params);
	m_consistentPhrases.AddHieroNonTerms(params);
	}

	void AlignedSentence::CreateConsistentPhrases(const Parameter &params)
	{
	int countT = m_target.size();
	int countS = m_source.size();

	m_consistentPhrases.Initialize(countS);

	// check alignments for target phrase startT...endT
	for(int lengthT=1;
	lengthT <= params.maxSpan && lengthT <= countT;
	lengthT++) {
	for(int startT=0; startT < countT-(lengthT-1); startT++) {

	// that's nice to have
	int endT = startT + lengthT - 1;

	// find find aligned source words
	// first: find minimum and maximum source word
	int minS = 9999;
	int maxS = -1;
	vector< int > usedS = GetSourceAlignmentCount();
	for(int ti=startT; ti<=endT; ti++) {
	const Word &word = *m_target[ti];
	const std::set<int> &alignment = word.GetAlignmentIndex();

	std::set<int>::const_iterator iterAlign;
	for(iterAlign = alignment.begin(); iterAlign != alignment.end(); ++iterAlign) {
	int si = *iterAlign;
	if (si<minS) {
	minS = si;
	}
	if (si>maxS) {
	maxS = si;
	}
	usedS[ si ]--;
	}
	}

	// unaligned phrases are not allowed
	if( maxS == -1 )
	continue;

	// source phrase has to be within limits
	size_t width = maxS - minS + 1;

	if( width < params.minSpan )
	continue;

	if( width > params.maxSpan )
	continue;

	// check if source words are aligned to out of bound target words
	bool out_of_bounds = false;
	for(int si=minS; si<=maxS && !out_of_bounds; si++)
	if (usedS[si]>0) {
	out_of_bounds = true;
	}

	// if out of bound, you gotta go
	if (out_of_bounds)
	continue;

	// done with all the checks, lets go over all consistent phrase pairs
	// start point of source phrase may retreat over unaligned
	for(int startS=minS;
	(startS>=0 &&
	startS>maxS - params.maxSpan && // within length limit
	(startS==minS \|\| m_source[startS]->GetAlignment().size()==0)); // unaligned
	startS--) {
	// end point of source phrase may advance over unaligned
	for(int endS=maxS;
	(endS<countS && endS<startS + params.maxSpan && // within length limit
	(endS==maxS \|\| m_source[endS]->GetAlignment().size()==0)); // unaligned
	endS++) {

	// take note that this is a valid phrase alignment
	m_consistentPhrases.Add(startS, endS, startT, endT, params);
	}
	}
	}
	}
	}