Add files using upload-large-folder tool

99f07cf verified over 1 year ago

17.3 kB

/*
 * Rule.cpp
 *
 *  Created on: 20 Feb 2014
 *      Author: hieu
 */

#include <sstream>
#include <algorithm>
#include "Rule.h"
#include "AlignedSentence.h"
#include "ConsistentPhrase.h"
#include "NonTerm.h"
#include "Parameter.h"

using namespace std;

Rule::Rule(const NonTerm &lhsNonTerm, const AlignedSentence &alignedSentence)
  :m_lhs(lhsNonTerm)
  ,m_alignedSentence(alignedSentence)
  ,m_isValid(true)
  ,m_canRecurse(true)
{
  CreateSource();
}

Rule::Rule(const Rule &copy, const NonTerm &nonTerm)
  :m_lhs(copy.m_lhs)
  ,m_alignedSentence(copy.m_alignedSentence)
  ,m_nonterms(copy.m_nonterms)
  ,m_isValid(true)
  ,m_canRecurse(true)
{
  m_nonterms.push_back(&nonTerm);
  CreateSource();

}

Rule::~Rule()
{
  // TODO Auto-generated destructor stub
}

const ConsistentPhrase &Rule::GetConsistentPhrase() const
{
  return m_lhs.GetConsistentPhrase();
}

void Rule::CreateSource()
{
  const NonTerm *cp = NULL;
  size_t nonTermInd = 0;
  if (nonTermInd < m_nonterms.size()) {
    cp = m_nonterms[nonTermInd];
  }

  for (int sourcePos = m_lhs.GetConsistentPhrase().corners[0];
       sourcePos <= m_lhs.GetConsistentPhrase().corners[1];
       ++sourcePos) {

    const RuleSymbol *ruleSymbol;
    if (cp && cp->GetConsistentPhrase().corners[0] <= sourcePos && sourcePos <= cp->GetConsistentPhrase().corners[1]) {
      // replace words with non-term
      ruleSymbol = cp;
      sourcePos = cp->GetConsistentPhrase().corners[1];
      if (m_nonterms.size()) {
        cp = m_nonterms[nonTermInd];
      }

      // move to next non-term
      ++nonTermInd;
      cp = (nonTermInd < m_nonterms.size()) ? m_nonterms[nonTermInd] : NULL;
    } else {
      // terminal
      ruleSymbol = m_alignedSentence.GetPhrase(Moses::Input)[sourcePos];
    }

    m_source.Add(ruleSymbol);
  }
}

int Rule::GetNextSourcePosForNonTerm() const
{
  if (m_nonterms.empty()) {
    // no non-terms so far. Can start next non-term on left corner
    return m_lhs.GetConsistentPhrase().corners[0];
  } else {
    // next non-term can start just left of previous
    const ConsistentPhrase &cp = m_nonterms.back()->GetConsistentPhrase();
    int nextPos = cp.corners[1] + 1;
    return nextPos;
  }
}

std::string Rule::Debug() const
{
  stringstream out;

  // source
  for (size_t i =  0; i < m_source.GetSize(); ++i) {
    const RuleSymbol &symbol = *m_source[i];
    out << symbol.Debug() << " ";
  }

  // target
  out << "||| ";
  for (size_t i =  0; i < m_target.GetSize(); ++i) {
    const RuleSymbol &symbol = *m_target[i];
    out << symbol.Debug() << " ";
  }

  out << "||| ";
  Alignments::const_iterator iterAlign;
  for (iterAlign =  m_alignments.begin(); iterAlign != m_alignments.end(); ++iterAlign) {
    const std::pair<int,int> &alignPair = *iterAlign;
    out << alignPair.first << "-" << alignPair.second << " ";
  }

  // overall range
  out << "||| LHS=" << m_lhs.Debug();

  return out.str();
}

void Rule::Output(std::ostream &out, bool forward) const
{
  if (forward) {
    // source
    m_source.Output(out);
    m_lhs.Output(out, Moses::Input);

    out << " ||| ";

    // target
    m_target.Output(out);
    m_lhs.Output(out, Moses::Output);
  } else {
    // target
    m_target.Output(out);
    m_lhs.Output(out, Moses::Output);

    out << " ||| ";

    // source
    m_source.Output(out);
    m_lhs.Output(out, Moses::Input);
  }

  out << " ||| ";

  // alignment
  Alignments::const_iterator iterAlign;
  for (iterAlign =  m_alignments.begin(); iterAlign != m_alignments.end(); ++iterAlign) {
    const std::pair<int,int> &alignPair = *iterAlign;

    if (forward) {
      out << alignPair.first << "-" << alignPair.second << " ";
    } else {
      out << alignPair.second << "-" << alignPair.first << " ";
    }
  }

  out << "||| ";

  // count
  out << m_count;

  out << " ||| ";

  // properties
  if (forward) {
    for (size_t i = 0; i < m_properties.size(); ++i) {
      const Property &prop = m_properties[i];
      out << prop << " ";
    }
  }
}

void Rule::NonTermContextFactor(int factor, const Word &word, std::ostream &out) const
{
  out << word.GetString(factor) << " ";
}

void Rule::NonTermContext(int sourceTarget, int factor, size_t ntInd, const ConsistentPhrase &cp, std::ostream &out) const
{
  int startPos, endPos;
  const Phrase *phrase;

  if (sourceTarget == 1) {
    startPos = cp.corners[0];
    endPos = cp.corners[1];
    phrase = &m_alignedSentence.GetPhrase(Moses::Input);
  } else if (sourceTarget == 2) {
    startPos = cp.corners[2];
    endPos = cp.corners[3];
    phrase = &m_alignedSentence.GetPhrase(Moses::Output);
  } else {
    abort();
  }

  out << ntInd << " ";

  // left outside
  if (startPos == 0) {
    out << "<s> ";
  } else {
    NonTermContextFactor(factor, *phrase->at(startPos - 1), out);
  }

  // left inside
  NonTermContextFactor(factor, *phrase->at(startPos), out);

  // right inside
  NonTermContextFactor(factor, *phrase->at(endPos), out);

  // right outside
  if (endPos == phrase->size() - 1) {
    out << "</s> ";
  } else {
    NonTermContextFactor(factor, *phrase->at(endPos + 1), out);
  }


}

void Rule::Prevalidate(const Parameter &params)
{
  // check number of source symbols in rule
  if (m_source.GetSize() > params.maxSymbolsSource) {
    m_isValid = false;
  }

  // check that last non-term added isn't too small
  if (m_nonterms.size()) {
    const NonTerm &lastNonTerm = *m_nonterms.back();
    const ConsistentPhrase &cp = lastNonTerm.GetConsistentPhrase();

    int sourceWidth = cp.GetWidth(Moses::Input);
    if (lastNonTerm.IsHiero(params)) {
      if (sourceWidth < params.minHoleSource) {
        m_isValid = false;
        m_canRecurse = false;
        return;
      }
    } else if (sourceWidth < params.minHoleSourceSyntax) {
      m_isValid = false;
      m_canRecurse = false;
      return;
    }

  }

  // check number of non-terms
  int numNonTerms = 0;
  int numHieroNonTerms = 0;
  for (size_t i = 0; i < m_source.GetSize(); ++i) {
    const RuleSymbol *arc = m_source[i];
    if (arc->IsNonTerm()) {
      ++numNonTerms;
      const NonTerm &nonTerm = *static_cast<const NonTerm*>(arc);
      bool isHiero = nonTerm.IsHiero(params);
      if (isHiero) {
        ++numHieroNonTerms;
      }
    }
  }

  if (numNonTerms >= params.maxNonTerm) {
    m_canRecurse = false;
    if (numNonTerms > params.maxNonTerm) {
      m_isValid = false;
      return;
    }
  }

  if (numHieroNonTerms >= params.maxHieroNonTerm) {
    m_canRecurse = false;
    if (numHieroNonTerms > params.maxHieroNonTerm) {
      m_isValid = false;
      return;
    }
  }

  // check if 2 consecutive non-terms in source
  if (!params.nonTermConsecSource && m_nonterms.size() >= 2) {
    const NonTerm &lastNonTerm = *m_nonterms.back();
    const NonTerm &secondLastNonTerm = *m_nonterms[m_nonterms.size() - 2];
    if (secondLastNonTerm.GetConsistentPhrase().corners[1] + 1 ==
        lastNonTerm.GetConsistentPhrase().corners[0]) {
      if (params.mixedSyntaxType == 0) {
        // ordinary hiero or syntax model
        m_isValid = false;
        m_canRecurse = false;
        return;
      } else {
        // Hieu's mixed syntax
        switch (params.nonTermConsecSourceMixedSyntax) {
        case 0:
          m_isValid = false;
          m_canRecurse = false;
          return;
        case 1:
          if (lastNonTerm.IsHiero(Moses::Input, params)
              && secondLastNonTerm.IsHiero(Moses::Input, params)) {
            m_isValid = false;
            m_canRecurse = false;
            return;
          }
          break;
        case 2:
          if (lastNonTerm.IsHiero(Moses::Input, params)
              || secondLastNonTerm.IsHiero(Moses::Input, params)) {
            m_isValid = false;
            m_canRecurse = false;
            return;
          }
          break;
        case 3:
          break;
        } // switch
      }
    }
  }

  //check to see if it overlaps with any other non-terms
  if (m_nonterms.size() >= 2) {
    const NonTerm &lastNonTerm = *m_nonterms.back();

    for (size_t i = 0; i < m_nonterms.size() - 1; ++i) {
      const NonTerm &otherNonTerm = *m_nonterms[i];
      bool overlap = lastNonTerm.GetConsistentPhrase().TargetOverlap(otherNonTerm.GetConsistentPhrase());

      if (overlap) {
        m_isValid = false;
        m_canRecurse = false;
        return;
      }
    }
  }

  // check that at least 1 word is aligned
  if (params.requireAlignedWord) {
    bool ok = false;
    for (size_t i = 0; i < m_source.GetSize(); ++i) {
      const RuleSymbol &symbol = *m_source[i];
      if (!symbol.IsNonTerm()) {
        const Word &word = static_cast<const Word&>(symbol);
        if (word.GetAlignment().size()) {
          ok = true;
          break;
        }
      }
    }

    if (!ok) {
      m_isValid = false;
      m_canRecurse = false;
      return;
    }
  }

  if (params.maxSpanFreeNonTermSource) {
    const NonTerm *front = dynamic_cast<const NonTerm*>(m_source[0]);
    if (front) {
      int width = front->GetWidth(Moses::Input);
      if (width > params.maxSpanFreeNonTermSource) {
        m_isValid = false;
        m_canRecurse = false;
        return;
      }
    }

    const NonTerm *back = dynamic_cast<const NonTerm*>(m_source.Back());
    if (back) {
      int width = back->GetWidth(Moses::Input);
      if (width > params.maxSpanFreeNonTermSource) {
        m_isValid = false;
        m_canRecurse = false;
        return;
      }
    }
  }

  if (!params.nieceTerminal) {
    // collect terminal in a rule
    std::set<const Word*> terms;
    for (size_t i = 0; i < m_source.GetSize(); ++i) {
      const Word *word = dynamic_cast<const Word*>(m_source[i]);
      if (word) {
        terms.insert(word);
      }
    }

    // look in non-terms
    for (size_t i = 0; i < m_source.GetSize(); ++i) {
      const NonTerm *nonTerm = dynamic_cast<const NonTerm*>(m_source[i]);
      if (nonTerm) {
        const ConsistentPhrase &cp = nonTerm->GetConsistentPhrase();
        bool containTerm = ContainTerm(cp, terms);

        if (containTerm) {
          //cerr << "ruleSource=" << *ruleSource << " ";
          //cerr << "ntRange=" << ntRange << endl;

          // non-term contains 1 of the terms in the rule.
          m_isValid = false;
          m_canRecurse = false;
          return;
        }
      }
    }
  }

  if (params.maxScope != UNDEFINED || params.minScope > 0) {
    int scope = GetScope(params);
    if (scope > params.maxScope) {
      // scope of subsequent rules will be the same or increase
      // therefore can NOT recurse
      m_isValid = false;
      m_canRecurse = false;
      return;
    }

    if (scope < params.minScope) {
      // scope of subsequent rules may increase
      // therefore can recurse
      m_isValid = false;
    }
  }

  // min/max span per scope
  if (params.scopeSpan.size()) {
    size_t scope = GetScope(params);
    if (scope >= params.scopeSpan.size()) {
      // no constraint on it. It's ok
    } else {
      const std::pair<int,int> &constraint = params.scopeSpan[scope];
      int sourceWidth = m_lhs.GetWidth(Moses::Input);
      if (sourceWidth < constraint.first || sourceWidth > constraint.second) {
        m_isValid = false;
        m_canRecurse = false;
        return;
      }
    }
  }
}

int Rule::GetScope(const Parameter &params) const
{
  size_t scope = 0;
  bool previousIsAmbiguous = false;

  if (m_source[0]->IsNonTerm()) {
    scope++;
    previousIsAmbiguous = true;
  }

  for (size_t i = 1; i < m_source.GetSize(); ++i) {
    const RuleSymbol *symbol = m_source[i];
    bool isAmbiguous = symbol->IsNonTerm();
    if (isAmbiguous) {
      // mixed syntax
      const NonTerm *nt = static_cast<const NonTerm*>(symbol);
      isAmbiguous = nt->IsHiero(Moses::Input, params);
    }

    if (isAmbiguous && previousIsAmbiguous) {
      scope++;
    }
    previousIsAmbiguous = isAmbiguous;
  }

  if (previousIsAmbiguous) {
    scope++;
  }

  return scope;

  /*
  int scope = 0;
  if (m_source.GetSize() > 1) {
    const RuleSymbol &front = *m_source.Front();
    if (front.IsNonTerm()) {
  	  ++scope;
    }
    const RuleSymbol &back = *m_source.Back();
    if (back.IsNonTerm()) {
  	  ++scope;
    }
  }
  return scope;
  */
}

template<typename T>
bool Contains(const T *sought, const set<const T*> &coll)
{
  std::set<const Word*>::const_iterator iter;
  for (iter = coll.begin(); iter != coll.end(); ++iter) {
    const Word *found = *iter;
    if (sought->CompareString(*found) == 0) {
      return true;
    }
  }
  return false;
}

bool Rule::ContainTerm(const ConsistentPhrase &cp, const std::set<const Word*> &terms) const
{
  const Phrase &sourceSentence = m_alignedSentence.GetPhrase(Moses::Input);

  for (int pos = cp.corners[0]; pos <= cp.corners[1]; ++pos) {
    const Word *soughtWord = sourceSentence[pos];

    // find same word in set
    if (Contains(soughtWord, terms)) {
      return true;
    }
  }
  return false;
}

bool CompareTargetNonTerms(const NonTerm *a, const NonTerm *b)
{
  // compare just start target pos
  return a->GetConsistentPhrase().corners[2] < b->GetConsistentPhrase().corners[2];
}

void Rule::CreateTarget(const Parameter &params)
{
  if (!m_isValid) {
    return;
  }

  vector<const NonTerm*> targetNonTerm(m_nonterms);
  std::sort(targetNonTerm.begin(), targetNonTerm.end(), CompareTargetNonTerms);

  const NonTerm *cp = NULL;
  size_t nonTermInd = 0;
  if (nonTermInd < targetNonTerm.size()) {
    cp = targetNonTerm[nonTermInd];
  }

  for (int targetPos = m_lhs.GetConsistentPhrase().corners[2];
       targetPos <= m_lhs.GetConsistentPhrase().corners[3];
       ++targetPos) {

    const RuleSymbol *ruleSymbol;
    if (cp && cp->GetConsistentPhrase().corners[2] <= targetPos && targetPos <= cp->GetConsistentPhrase().corners[3]) {
      // replace words with non-term
      ruleSymbol = cp;
      targetPos = cp->GetConsistentPhrase().corners[3];
      if (targetNonTerm.size()) {
        cp = targetNonTerm[nonTermInd];
      }

      // move to next non-term
      ++nonTermInd;
      cp = (nonTermInd < targetNonTerm.size()) ? targetNonTerm[nonTermInd] : NULL;
    } else {
      // terminal
      ruleSymbol = m_alignedSentence.GetPhrase(Moses::Output)[targetPos];
    }

    m_target.Add(ruleSymbol);
  }

  CreateAlignments();
}


void Rule::CreateAlignments()
{
  for (size_t sourcePos = 0; sourcePos < m_source.GetSize(); ++sourcePos) {
    const RuleSymbol *symbol = m_source[sourcePos];
    if (!symbol->IsNonTerm()) {
      // terminals
      const Word &sourceWord = static_cast<const Word&>(*symbol);
      const std::set<const Word *> &targetWords = sourceWord.GetAlignment();
      CreateAlignments(sourcePos, targetWords);
    } else {
      // non-terms. same object in both source & target
      CreateAlignments(sourcePos, symbol);
    }
  }
}

void Rule::CreateAlignments(int sourcePos, const std::set<const Word *> &targetWords)
{
  std::set<const Word *>::const_iterator iterTarget;
  for (iterTarget = targetWords.begin(); iterTarget != targetWords.end(); ++iterTarget) {
    const Word *targetWord = *iterTarget;
    CreateAlignments(sourcePos, targetWord);
  }
}

void Rule::CreateAlignments(int sourcePos, const RuleSymbol *targetSought)
{
  // should be in target phrase
  for (size_t targetPos = 0; targetPos < m_target.GetSize(); ++targetPos) {
    const RuleSymbol *foundSymbol = m_target[targetPos];
    if (targetSought == foundSymbol) {
      pair<int, int> alignPoint(sourcePos, targetPos);
      m_alignments.insert(alignPoint);
      return;
    }
  }

  throw "not found";
}

void Rule::CreateProperties(const Parameter &params)
{
  //cerr << Debug() << " " << m_nonterms.size() << endl;

  // span length
  if (params.spanLength && m_nonterms.size()) {
    stringstream strme;
    strme << "{{SpanLength ";

    for (size_t i = 0; i < m_nonterms.size(); ++i) {
      const NonTerm &nonTerm = *m_nonterms[i];
      const ConsistentPhrase &cp = nonTerm.GetConsistentPhrase();
      strme << i << "," << cp.GetWidth(Moses::Input) << "," << cp.GetWidth(Moses::Output) << " ";
    }
    strme << "}}";

    m_properties.push_back(strme.str());
  }

  if (params.ruleLength && m_nonterms.size()) {
    const ConsistentPhrase &cp = m_lhs.GetConsistentPhrase();

    stringstream strme;
    strme << "{{RuleLength ";
    strme << cp.GetWidth(Moses::Input);
    strme << "}}";

    m_properties.push_back(strme.str());
  }

  // non-term context (source)
  if (params.nonTermContext && m_nonterms.size()) {
    stringstream strme;
    strme << "{{NonTermContext ";

    int factor = params.nonTermContextFactor;

    for (size_t i = 0; i < m_nonterms.size(); ++i) {
      const NonTerm &nonTerm = *m_nonterms[i];
      const ConsistentPhrase &cp = nonTerm.GetConsistentPhrase();
      NonTermContext(1, factor, i, cp, strme);
    }
    strme << "}}";

    m_properties.push_back(strme.str());
  }

  // non-term context (target)
  if (params.nonTermContextTarget && m_nonterms.size()) {
    stringstream strme;
    strme << "{{NonTermContextTarget ";

    int factor = params.nonTermContextFactor;

    for (size_t i = 0; i < m_nonterms.size(); ++i) {
      const NonTerm &nonTerm = *m_nonterms[i];
      const ConsistentPhrase &cp = nonTerm.GetConsistentPhrase();
      NonTermContext(2, factor, i, cp, strme);
    }
    strme << "}}";

    m_properties.push_back(strme.str());
  }

}

	/*
	* Rule.cpp
	*
	* Created on: 20 Feb 2014
	* Author: hieu
	*/

	#include <sstream>
	#include <algorithm>
	#include "Rule.h"
	#include "AlignedSentence.h"
	#include "ConsistentPhrase.h"
	#include "NonTerm.h"
	#include "Parameter.h"

	using namespace std;

	Rule::Rule(const NonTerm &lhsNonTerm, const AlignedSentence &alignedSentence)
	:m_lhs(lhsNonTerm)
	,m_alignedSentence(alignedSentence)
	,m_isValid(true)
	,m_canRecurse(true)
	{
	CreateSource();
	}

	Rule::Rule(const Rule &copy, const NonTerm &nonTerm)
	:m_lhs(copy.m_lhs)
	,m_alignedSentence(copy.m_alignedSentence)
	,m_nonterms(copy.m_nonterms)
	,m_isValid(true)
	,m_canRecurse(true)
	{
	m_nonterms.push_back(&nonTerm);
	CreateSource();

	}

	Rule::~Rule()
	{
	// TODO Auto-generated destructor stub
	}

	const ConsistentPhrase &Rule::GetConsistentPhrase() const
	{
	return m_lhs.GetConsistentPhrase();
	}

	void Rule::CreateSource()
	{
	const NonTerm *cp = NULL;
	size_t nonTermInd = 0;
	if (nonTermInd < m_nonterms.size()) {
	cp = m_nonterms[nonTermInd];
	}

	for (int sourcePos = m_lhs.GetConsistentPhrase().corners[0];
	sourcePos <= m_lhs.GetConsistentPhrase().corners[1];
	++sourcePos) {

	const RuleSymbol *ruleSymbol;
	if (cp && cp->GetConsistentPhrase().corners[0] <= sourcePos && sourcePos <= cp->GetConsistentPhrase().corners[1]) {
	// replace words with non-term
	ruleSymbol = cp;
	sourcePos = cp->GetConsistentPhrase().corners[1];
	if (m_nonterms.size()) {
	cp = m_nonterms[nonTermInd];
	}

	// move to next non-term
	++nonTermInd;
	cp = (nonTermInd < m_nonterms.size()) ? m_nonterms[nonTermInd] : NULL;
	} else {
	// terminal
	ruleSymbol = m_alignedSentence.GetPhrase(Moses::Input)[sourcePos];
	}

	m_source.Add(ruleSymbol);
	}
	}

	int Rule::GetNextSourcePosForNonTerm() const
	{
	if (m_nonterms.empty()) {
	// no non-terms so far. Can start next non-term on left corner
	return m_lhs.GetConsistentPhrase().corners[0];
	} else {
	// next non-term can start just left of previous
	const ConsistentPhrase &cp = m_nonterms.back()->GetConsistentPhrase();
	int nextPos = cp.corners[1] + 1;
	return nextPos;
	}
	}

	std::string Rule::Debug() const
	{
	stringstream out;

	// source
	for (size_t i = 0; i < m_source.GetSize(); ++i) {
	const RuleSymbol &symbol = *m_source[i];
	out << symbol.Debug() << " ";
	}

	// target
	out << "\|\|\| ";
	for (size_t i = 0; i < m_target.GetSize(); ++i) {
	const RuleSymbol &symbol = *m_target[i];
	out << symbol.Debug() << " ";
	}

	out << "\|\|\| ";
	Alignments::const_iterator iterAlign;
	for (iterAlign = m_alignments.begin(); iterAlign != m_alignments.end(); ++iterAlign) {
	const std::pair<int,int> &alignPair = *iterAlign;
	out << alignPair.first << "-" << alignPair.second << " ";
	}

	// overall range
	out << "\|\|\| LHS=" << m_lhs.Debug();

	return out.str();
	}

	void Rule::Output(std::ostream &out, bool forward) const
	{
	if (forward) {
	// source
	m_source.Output(out);
	m_lhs.Output(out, Moses::Input);

	out << " \|\|\| ";

	// target
	m_target.Output(out);
	m_lhs.Output(out, Moses::Output);
	} else {
	// target
	m_target.Output(out);
	m_lhs.Output(out, Moses::Output);

	out << " \|\|\| ";

	// source
	m_source.Output(out);
	m_lhs.Output(out, Moses::Input);
	}

	out << " \|\|\| ";

	// alignment
	Alignments::const_iterator iterAlign;
	for (iterAlign = m_alignments.begin(); iterAlign != m_alignments.end(); ++iterAlign) {
	const std::pair<int,int> &alignPair = *iterAlign;

	if (forward) {
	out << alignPair.first << "-" << alignPair.second << " ";
	} else {
	out << alignPair.second << "-" << alignPair.first << " ";
	}
	}

	out << "\|\|\| ";

	// count
	out << m_count;

	out << " \|\|\| ";

	// properties
	if (forward) {
	for (size_t i = 0; i < m_properties.size(); ++i) {
	const Property &prop = m_properties[i];
	out << prop << " ";
	}
	}
	}

	void Rule::NonTermContextFactor(int factor, const Word &word, std::ostream &out) const
	{
	out << word.GetString(factor) << " ";
	}

	void Rule::NonTermContext(int sourceTarget, int factor, size_t ntInd, const ConsistentPhrase &cp, std::ostream &out) const
	{
	int startPos, endPos;
	const Phrase *phrase;

	if (sourceTarget == 1) {
	startPos = cp.corners[0];
	endPos = cp.corners[1];
	phrase = &m_alignedSentence.GetPhrase(Moses::Input);
	} else if (sourceTarget == 2) {
	startPos = cp.corners[2];
	endPos = cp.corners[3];
	phrase = &m_alignedSentence.GetPhrase(Moses::Output);
	} else {
	abort();
	}

	out << ntInd << " ";

	// left outside
	if (startPos == 0) {
	out << "<s> ";
	} else {
	NonTermContextFactor(factor, *phrase->at(startPos - 1), out);
	}

	// left inside
	NonTermContextFactor(factor, *phrase->at(startPos), out);

	// right inside
	NonTermContextFactor(factor, *phrase->at(endPos), out);

	// right outside
	if (endPos == phrase->size() - 1) {
	out << "</s> ";
	} else {
	NonTermContextFactor(factor, *phrase->at(endPos + 1), out);
	}


	}

	void Rule::Prevalidate(const Parameter &params)
	{
	// check number of source symbols in rule
	if (m_source.GetSize() > params.maxSymbolsSource) {
	m_isValid = false;
	}

	// check that last non-term added isn't too small
	if (m_nonterms.size()) {
	const NonTerm &lastNonTerm = *m_nonterms.back();
	const ConsistentPhrase &cp = lastNonTerm.GetConsistentPhrase();

	int sourceWidth = cp.GetWidth(Moses::Input);
	if (lastNonTerm.IsHiero(params)) {
	if (sourceWidth < params.minHoleSource) {
	m_isValid = false;
	m_canRecurse = false;
	return;
	}
	} else if (sourceWidth < params.minHoleSourceSyntax) {
	m_isValid = false;
	m_canRecurse = false;
	return;
	}

	}

	// check number of non-terms
	int numNonTerms = 0;
	int numHieroNonTerms = 0;
	for (size_t i = 0; i < m_source.GetSize(); ++i) {
	const RuleSymbol *arc = m_source[i];
	if (arc->IsNonTerm()) {
	++numNonTerms;
	const NonTerm &nonTerm = static_cast<const NonTerm>(arc);
	bool isHiero = nonTerm.IsHiero(params);
	if (isHiero) {
	++numHieroNonTerms;
	}
	}
	}

	if (numNonTerms >= params.maxNonTerm) {
	m_canRecurse = false;
	if (numNonTerms > params.maxNonTerm) {
	m_isValid = false;
	return;
	}
	}

	if (numHieroNonTerms >= params.maxHieroNonTerm) {
	m_canRecurse = false;
	if (numHieroNonTerms > params.maxHieroNonTerm) {
	m_isValid = false;
	return;
	}
	}

	// check if 2 consecutive non-terms in source
	if (!params.nonTermConsecSource && m_nonterms.size() >= 2) {
	const NonTerm &lastNonTerm = *m_nonterms.back();
	const NonTerm &secondLastNonTerm = *m_nonterms[m_nonterms.size() - 2];
	if (secondLastNonTerm.GetConsistentPhrase().corners[1] + 1 ==
	lastNonTerm.GetConsistentPhrase().corners[0]) {
	if (params.mixedSyntaxType == 0) {
	// ordinary hiero or syntax model
	m_isValid = false;
	m_canRecurse = false;
	return;
	} else {
	// Hieu's mixed syntax
	switch (params.nonTermConsecSourceMixedSyntax) {
	case 0:
	m_isValid = false;
	m_canRecurse = false;
	return;
	case 1:
	if (lastNonTerm.IsHiero(Moses::Input, params)
	&& secondLastNonTerm.IsHiero(Moses::Input, params)) {
	m_isValid = false;
	m_canRecurse = false;
	return;
	}
	break;
	case 2:
	if (lastNonTerm.IsHiero(Moses::Input, params)
	\|\| secondLastNonTerm.IsHiero(Moses::Input, params)) {
	m_isValid = false;
	m_canRecurse = false;
	return;
	}
	break;
	case 3:
	break;
	} // switch
	}
	}
	}

	//check to see if it overlaps with any other non-terms
	if (m_nonterms.size() >= 2) {
	const NonTerm &lastNonTerm = *m_nonterms.back();

	for (size_t i = 0; i < m_nonterms.size() - 1; ++i) {
	const NonTerm &otherNonTerm = *m_nonterms[i];
	bool overlap = lastNonTerm.GetConsistentPhrase().TargetOverlap(otherNonTerm.GetConsistentPhrase());

	if (overlap) {
	m_isValid = false;
	m_canRecurse = false;
	return;
	}
	}
	}

	// check that at least 1 word is aligned
	if (params.requireAlignedWord) {
	bool ok = false;
	for (size_t i = 0; i < m_source.GetSize(); ++i) {
	const RuleSymbol &symbol = *m_source[i];
	if (!symbol.IsNonTerm()) {
	const Word &word = static_cast<const Word&>(symbol);
	if (word.GetAlignment().size()) {
	ok = true;
	break;
	}
	}
	}

	if (!ok) {
	m_isValid = false;
	m_canRecurse = false;
	return;
	}
	}

	if (params.maxSpanFreeNonTermSource) {
	const NonTerm front = dynamic_cast<const NonTerm>(m_source[0]);
	if (front) {
	int width = front->GetWidth(Moses::Input);
	if (width > params.maxSpanFreeNonTermSource) {
	m_isValid = false;
	m_canRecurse = false;
	return;
	}
	}

	const NonTerm back = dynamic_cast<const NonTerm>(m_source.Back());
	if (back) {
	int width = back->GetWidth(Moses::Input);
	if (width > params.maxSpanFreeNonTermSource) {
	m_isValid = false;
	m_canRecurse = false;
	return;
	}
	}
	}

	if (!params.nieceTerminal) {
	// collect terminal in a rule
	std::set<const Word*> terms;
	for (size_t i = 0; i < m_source.GetSize(); ++i) {
	const Word word = dynamic_cast<const Word>(m_source[i]);
	if (word) {
	terms.insert(word);
	}
	}

	// look in non-terms
	for (size_t i = 0; i < m_source.GetSize(); ++i) {
	const NonTerm nonTerm = dynamic_cast<const NonTerm>(m_source[i]);
	if (nonTerm) {
	const ConsistentPhrase &cp = nonTerm->GetConsistentPhrase();
	bool containTerm = ContainTerm(cp, terms);

	if (containTerm) {
	//cerr << "ruleSource=" << *ruleSource << " ";
	//cerr << "ntRange=" << ntRange << endl;

	// non-term contains 1 of the terms in the rule.
	m_isValid = false;
	m_canRecurse = false;
	return;
	}
	}
	}
	}

	if (params.maxScope != UNDEFINED \|\| params.minScope > 0) {
	int scope = GetScope(params);
	if (scope > params.maxScope) {
	// scope of subsequent rules will be the same or increase
	// therefore can NOT recurse
	m_isValid = false;
	m_canRecurse = false;
	return;
	}

	if (scope < params.minScope) {
	// scope of subsequent rules may increase
	// therefore can recurse
	m_isValid = false;
	}
	}

	// min/max span per scope
	if (params.scopeSpan.size()) {
	size_t scope = GetScope(params);
	if (scope >= params.scopeSpan.size()) {
	// no constraint on it. It's ok
	} else {
	const std::pair<int,int> &constraint = params.scopeSpan[scope];
	int sourceWidth = m_lhs.GetWidth(Moses::Input);
	if (sourceWidth < constraint.first \|\| sourceWidth > constraint.second) {
	m_isValid = false;
	m_canRecurse = false;
	return;
	}
	}
	}
	}

	int Rule::GetScope(const Parameter &params) const
	{
	size_t scope = 0;
	bool previousIsAmbiguous = false;

	if (m_source[0]->IsNonTerm()) {
	scope++;
	previousIsAmbiguous = true;
	}

	for (size_t i = 1; i < m_source.GetSize(); ++i) {
	const RuleSymbol *symbol = m_source[i];
	bool isAmbiguous = symbol->IsNonTerm();
	if (isAmbiguous) {
	// mixed syntax
	const NonTerm nt = static_cast<const NonTerm>(symbol);
	isAmbiguous = nt->IsHiero(Moses::Input, params);
	}

	if (isAmbiguous && previousIsAmbiguous) {
	scope++;
	}
	previousIsAmbiguous = isAmbiguous;
	}

	if (previousIsAmbiguous) {
	scope++;
	}

	return scope;

	/*
	int scope = 0;
	if (m_source.GetSize() > 1) {
	const RuleSymbol &front = *m_source.Front();
	if (front.IsNonTerm()) {
	++scope;
	}

	const RuleSymbol &back = *m_source.Back();
	if (back.IsNonTerm()) {
	++scope;
	}
	}
	return scope;
	*/
	}

	template<typename T>
	bool Contains(const T sought, const set<const T> &coll)
	{
	std::set<const Word*>::const_iterator iter;
	for (iter = coll.begin(); iter != coll.end(); ++iter) {
	const Word found = iter;
	if (sought->CompareString(*found) == 0) {
	return true;
	}
	}
	return false;
	}

	bool Rule::ContainTerm(const ConsistentPhrase &cp, const std::set<const Word*> &terms) const
	{
	const Phrase &sourceSentence = m_alignedSentence.GetPhrase(Moses::Input);

	for (int pos = cp.corners[0]; pos <= cp.corners[1]; ++pos) {
	const Word *soughtWord = sourceSentence[pos];

	// find same word in set
	if (Contains(soughtWord, terms)) {
	return true;
	}
	}
	return false;
	}

	bool CompareTargetNonTerms(const NonTerm a, const NonTerm b)
	{
	// compare just start target pos
	return a->GetConsistentPhrase().corners[2] < b->GetConsistentPhrase().corners[2];
	}

	void Rule::CreateTarget(const Parameter &params)
	{
	if (!m_isValid) {
	return;
	}

	vector<const NonTerm*> targetNonTerm(m_nonterms);
	std::sort(targetNonTerm.begin(), targetNonTerm.end(), CompareTargetNonTerms);

	const NonTerm *cp = NULL;
	size_t nonTermInd = 0;
	if (nonTermInd < targetNonTerm.size()) {
	cp = targetNonTerm[nonTermInd];
	}

	for (int targetPos = m_lhs.GetConsistentPhrase().corners[2];
	targetPos <= m_lhs.GetConsistentPhrase().corners[3];
	++targetPos) {

	const RuleSymbol *ruleSymbol;
	if (cp && cp->GetConsistentPhrase().corners[2] <= targetPos && targetPos <= cp->GetConsistentPhrase().corners[3]) {
	// replace words with non-term
	ruleSymbol = cp;
	targetPos = cp->GetConsistentPhrase().corners[3];
	if (targetNonTerm.size()) {
	cp = targetNonTerm[nonTermInd];
	}

	// move to next non-term
	++nonTermInd;
	cp = (nonTermInd < targetNonTerm.size()) ? targetNonTerm[nonTermInd] : NULL;
	} else {
	// terminal
	ruleSymbol = m_alignedSentence.GetPhrase(Moses::Output)[targetPos];
	}

	m_target.Add(ruleSymbol);
	}

	CreateAlignments();
	}


	void Rule::CreateAlignments()
	{
	for (size_t sourcePos = 0; sourcePos < m_source.GetSize(); ++sourcePos) {
	const RuleSymbol *symbol = m_source[sourcePos];
	if (!symbol->IsNonTerm()) {
	// terminals
	const Word &sourceWord = static_cast<const Word&>(*symbol);
	const std::set<const Word *> &targetWords = sourceWord.GetAlignment();
	CreateAlignments(sourcePos, targetWords);
	} else {
	// non-terms. same object in both source & target
	CreateAlignments(sourcePos, symbol);
	}
	}
	}

	void Rule::CreateAlignments(int sourcePos, const std::set<const Word *> &targetWords)
	{
	std::set<const Word *>::const_iterator iterTarget;
	for (iterTarget = targetWords.begin(); iterTarget != targetWords.end(); ++iterTarget) {
	const Word targetWord = iterTarget;
	CreateAlignments(sourcePos, targetWord);
	}
	}

	void Rule::CreateAlignments(int sourcePos, const RuleSymbol *targetSought)
	{
	// should be in target phrase
	for (size_t targetPos = 0; targetPos < m_target.GetSize(); ++targetPos) {
	const RuleSymbol *foundSymbol = m_target[targetPos];
	if (targetSought == foundSymbol) {
	pair<int, int> alignPoint(sourcePos, targetPos);
	m_alignments.insert(alignPoint);
	return;
	}
	}

	throw "not found";
	}

	void Rule::CreateProperties(const Parameter &params)
	{
	//cerr << Debug() << " " << m_nonterms.size() << endl;

	// span length
	if (params.spanLength && m_nonterms.size()) {
	stringstream strme;
	strme << "{{SpanLength ";

	for (size_t i = 0; i < m_nonterms.size(); ++i) {
	const NonTerm &nonTerm = *m_nonterms[i];
	const ConsistentPhrase &cp = nonTerm.GetConsistentPhrase();
	strme << i << "," << cp.GetWidth(Moses::Input) << "," << cp.GetWidth(Moses::Output) << " ";
	}
	strme << "}}";

	m_properties.push_back(strme.str());
	}

	if (params.ruleLength && m_nonterms.size()) {
	const ConsistentPhrase &cp = m_lhs.GetConsistentPhrase();

	stringstream strme;
	strme << "{{RuleLength ";
	strme << cp.GetWidth(Moses::Input);
	strme << "}}";

	m_properties.push_back(strme.str());
	}

	// non-term context (source)
	if (params.nonTermContext && m_nonterms.size()) {
	stringstream strme;
	strme << "{{NonTermContext ";

	int factor = params.nonTermContextFactor;

	for (size_t i = 0; i < m_nonterms.size(); ++i) {
	const NonTerm &nonTerm = *m_nonterms[i];
	const ConsistentPhrase &cp = nonTerm.GetConsistentPhrase();
	NonTermContext(1, factor, i, cp, strme);
	}
	strme << "}}";

	m_properties.push_back(strme.str());
	}

	// non-term context (target)
	if (params.nonTermContextTarget && m_nonterms.size()) {
	stringstream strme;
	strme << "{{NonTermContextTarget ";

	int factor = params.nonTermContextFactor;

	for (size_t i = 0; i < m_nonterms.size(); ++i) {
	const NonTerm &nonTerm = *m_nonterms[i];
	const ConsistentPhrase &cp = nonTerm.GetConsistentPhrase();
	NonTermContext(2, factor, i, cp, strme);
	}
	strme << "}}";

	m_properties.push_back(strme.str());
	}

	}