wiki-cased / tools /mosesdecoder /moses /FF /DistortionScoreProducer.cpp

Upload folder using huggingface_hub

41f6dd8 verified almost 2 years ago

10.4 kB

#include "DistortionScoreProducer.h"
#include "FFState.h"
#include "moses/InputPath.h"
#include "moses/Range.h"
#include "moses/StaticData.h"
#include "moses/Hypothesis.h"
#include "moses/Manager.h"
#include "moses/FactorCollection.h"
#include <cmath>

using namespace std;

namespace Moses
{
struct DistortionState : public FFState {
  Range range;
  int first_gap;
  bool inSubordinateConjunction;
  DistortionState(const Range& wr, int fg, bool subord=false) : range(wr), first_gap(fg), inSubordinateConjunction(subord) {}

  size_t hash() const {
    return range.GetEndPos();
  }
  virtual bool operator==(const FFState& other) const {
    const DistortionState& o =
      static_cast<const DistortionState&>(other);
    return ( (range.GetEndPos() == o.range.GetEndPos()) && (inSubordinateConjunction == o.inSubordinateConjunction) );
  }

};

std::vector<const DistortionScoreProducer*> DistortionScoreProducer::s_staticColl;

DistortionScoreProducer::DistortionScoreProducer(const std::string &line)
  : StatefulFeatureFunction(1, line)
  , m_useSparse(false)
  , m_sparseDistance(false)
  , m_sparseSubordinate(false)
{
  s_staticColl.push_back(this);
  ReadParameters();
}

void DistortionScoreProducer::SetParameter(const std::string& key, const std::string& value)
{
  if (key == "sparse") {
    m_useSparse = Scan<bool>(value);
  } else if (key == "sparse-distance") {
    m_sparseDistance = Scan<bool>(value);
  } else if (key == "sparse-input-factor") {
    m_sparseFactorTypeSource = Scan<FactorType>(value);
  } else if (key == "sparse-output-factor") {
    m_sparseFactorTypeTarget = Scan<FactorType>(value);
  } else if (key == "sparse-subordinate") {
    std::string subordinateConjunctionTag = Scan<std::string>(value);
    FactorCollection &factorCollection = FactorCollection::Instance();
    m_subordinateConjunctionTagFactor = factorCollection.AddFactor(subordinateConjunctionTag,false);
    m_sparseSubordinate = true;
  } else if (key == "sparse-subordinate-output-factor") {
    m_sparseFactorTypeTargetSubordinate = Scan<FactorType>(value);
  } else {
    StatefulFeatureFunction::SetParameter(key, value);
  }
}

const FFState* DistortionScoreProducer::EmptyHypothesisState(const InputType &input) const
{
  // fake previous translated phrase start and end
  size_t start = NOT_FOUND;
  size_t end = NOT_FOUND;
  if (input.m_frontSpanCoveredLength > 0) {
    // can happen with --continue-partial-translation
    start = 0;
    end = input.m_frontSpanCoveredLength -1;
  }
  return new DistortionState(
           Range(start, end),
           NOT_FOUND);
}

float
DistortionScoreProducer::
CalculateDistortionScore(const Hypothesis& hypo,
                         const Range &prev, const Range &curr, const int FirstGap)
{
  // if(!StaticData::Instance().UseEarlyDistortionCost()) {
  if(!hypo.GetManager().options()->reordering.use_early_distortion_cost) {
    return - (float) hypo.GetInput().ComputeDistortionDistance(prev, curr);
  } // else {

  /* Pay distortion score as soon as possible, from Moore and Quirk MT Summit 2007
     Definitions:
     S   : current source range
     S'  : last translated source phrase range
     S'' : longest fully-translated initial segment
  */

  int prefixEndPos = (int)FirstGap-1;
  if((int)FirstGap==-1)
    prefixEndPos = -1;

  // case1: S is adjacent to S'' => return 0
  if ((int) curr.GetStartPos() == prefixEndPos+1) {
    IFVERBOSE(4) std::cerr<< "MQ07disto:case1" << std::endl;
    return 0;
  }

  // case2: S is to the left of S' => return 2(length(S))
  if ((int) curr.GetEndPos() < (int) prev.GetEndPos()) {
    IFVERBOSE(4) std::cerr<< "MQ07disto:case2" << std::endl;
    return (float) -2*(int)curr.GetNumWordsCovered();
  }

  // case3: S' is a subsequence of S'' => return 2(nbWordBetween(S,S'')+length(S))
  if ((int) prev.GetEndPos() <= prefixEndPos) {
    IFVERBOSE(4) std::cerr<< "MQ07disto:case3" << std::endl;
    int z = (int)curr.GetStartPos()-prefixEndPos - 1;
    return (float) -2*(z + (int)curr.GetNumWordsCovered());
  }

  // case4: otherwise => return 2(nbWordBetween(S,S')+length(S))
  IFVERBOSE(4) std::cerr<< "MQ07disto:case4" << std::endl;
  return (float) -2*((int)curr.GetNumWordsBetween(prev) + (int)curr.GetNumWordsCovered());

}


FFState* DistortionScoreProducer::EvaluateWhenApplied(
  const Hypothesis& hypo,
  const FFState* prev_state,
  ScoreComponentCollection* out) const
{
  const DistortionState* prev = static_cast<const DistortionState*>(prev_state);
  bool subordinateConjunction = prev->inSubordinateConjunction;

  if (m_useSparse) {
    int jumpFromPos = prev->range.GetEndPos()+1;
    int jumpToPos = hypo.GetCurrSourceWordsRange().GetStartPos();
    size_t distance = std::abs( jumpFromPos - jumpToPos );

    const Sentence& sentence = static_cast<const Sentence&>(hypo.GetInput());

    StringPiece jumpFromSourceFactorPrev;
    StringPiece jumpFromSourceFactor;
    StringPiece jumpToSourceFactor;
    if (jumpFromPos < (int)sentence.GetSize()) {
      jumpFromSourceFactor = sentence.GetWord(jumpFromPos).GetFactor(m_sparseFactorTypeSource)->GetString();
    } else {
      jumpFromSourceFactor = "</s>";
    }
    if (jumpFromPos > 0) {
      jumpFromSourceFactorPrev = sentence.GetWord(jumpFromPos-1).GetFactor(m_sparseFactorTypeSource)->GetString();
    } else {
      jumpFromSourceFactorPrev = "<s>";
    }
    jumpToSourceFactor = sentence.GetWord(jumpToPos).GetFactor(m_sparseFactorTypeSource)->GetString();

    const TargetPhrase& currTargetPhrase = hypo.GetCurrTargetPhrase();
    StringPiece jumpToTargetFactor = currTargetPhrase.GetWord(0).GetFactor(m_sparseFactorTypeTarget)->GetString();

    util::StringStream featureName;

    // source factor (start position)
    featureName = util::StringStream();
    featureName << m_description << "_";
    if ( jumpToPos > jumpFromPos ) {
      featureName << "R";
    } else if ( jumpToPos < jumpFromPos ) {
      featureName << "L";
    } else {
      featureName << "M";
    }
    if (m_sparseDistance) {
      featureName << distance;
    }
    featureName << "_SFS_" << jumpFromSourceFactor;
    if (m_sparseSubordinate && subordinateConjunction) {
      featureName << "_SUBORD";
    }
    out->SparsePlusEquals(featureName.str(), 1);

    // source factor (start position minus 1)
    featureName = util::StringStream();
    featureName << m_description << "_";
    if ( jumpToPos > jumpFromPos ) {
      featureName << "R";
    } else if ( jumpToPos < jumpFromPos ) {
      featureName << "L";
    } else {
      featureName << "M";
    }
    if (m_sparseDistance) {
      featureName << distance;
    }
    featureName << "_SFP_" << jumpFromSourceFactorPrev;
    if (m_sparseSubordinate && subordinateConjunction) {
      featureName << "_SUBORD";
    }
    out->SparsePlusEquals(featureName.str(), 1);

    // source factor (end position)
    featureName = util::StringStream();
    featureName << m_description << "_";
    if ( jumpToPos > jumpFromPos ) {
      featureName << "R";
    } else if ( jumpToPos < jumpFromPos ) {
      featureName << "L";
    } else {
      featureName << "M";
    }
    if (m_sparseDistance) {
      featureName << distance;
    }
    featureName << "_SFE_" << jumpToSourceFactor;
    if (m_sparseSubordinate && subordinateConjunction) {
      featureName << "_SUBORD";
    }
    out->SparsePlusEquals(featureName.str(), 1);

    // target factor (end position)
    featureName = util::StringStream();
    featureName << m_description << "_";
    if ( jumpToPos > jumpFromPos ) {
      featureName << "R";
    } else if ( jumpToPos < jumpFromPos ) {
      featureName << "L";
    } else {
      featureName << "M";
    }
    if (m_sparseDistance) {
      featureName << distance;
    }
    featureName << "_TFE_" << jumpToTargetFactor;
    if (m_sparseSubordinate && subordinateConjunction) {
      featureName << "_SUBORD";
    }
    out->SparsePlusEquals(featureName.str(), 1);

    // relative source sentence position
    featureName = util::StringStream();
    featureName << m_description << "_";
    if ( jumpToPos > jumpFromPos ) {
      featureName << "R";
    } else if ( jumpToPos < jumpFromPos ) {
      featureName << "L";
    } else {
      featureName << "M";
    }
    if (m_sparseDistance) {
      featureName << distance;
    }
    size_t relativeSourceSentencePosBin = std::floor( 5 * (float)jumpFromPos / (sentence.GetSize()+1) );
    featureName << "_P_" << relativeSourceSentencePosBin;
    if (m_sparseSubordinate && subordinateConjunction) {
      featureName << "_SUBORD";
    }
    out->SparsePlusEquals(featureName.str(), 1);

    // source sentence length bin
    featureName = util::StringStream();
    featureName << m_description << "_";
    if ( jumpToPos > jumpFromPos ) {
      featureName << "R";
    } else if ( jumpToPos < jumpFromPos ) {
      featureName << "L";
    } else {
      featureName << "M";
    }
    if (m_sparseDistance) {
      featureName << distance;
    }
    size_t sourceSentenceLengthBin = 3;
    if (sentence.GetSize() < 15) {
      sourceSentenceLengthBin = 0;
    } else if (sentence.GetSize() < 23) {
      sourceSentenceLengthBin = 1;
    } else if (sentence.GetSize() < 33) {
      sourceSentenceLengthBin = 2;
    }
    featureName << "_SL_" << sourceSentenceLengthBin;
    if (m_sparseSubordinate && subordinateConjunction) {
      featureName << "_SUBORD";
    }
    out->SparsePlusEquals(featureName.str(), 1);

    if (m_sparseSubordinate) {
      for (size_t posT=0; posT<currTargetPhrase.GetSize(); ++posT) {
        const Word &wordT = currTargetPhrase.GetWord(posT);
        if (wordT[m_sparseFactorTypeTargetSubordinate] == m_subordinateConjunctionTagFactor) {
          subordinateConjunction = true;
        } else if (wordT[m_sparseFactorTypeTargetSubordinate]->GetString()[0] == 'V') {
          subordinateConjunction = false;
        }
      };
    }
  }

  const float distortionScore = CalculateDistortionScore(
                                  hypo,
                                  prev->range,
                                  hypo.GetCurrSourceWordsRange(),
                                  prev->first_gap);
  out->PlusEquals(this, distortionScore);

  DistortionState* state = new DistortionState(
    hypo.GetCurrSourceWordsRange(),
    hypo.GetWordsBitmap().GetFirstGapPos(),
    subordinateConjunction);

  return state;
}


}


	#include "DistortionScoreProducer.h"
	#include "FFState.h"
	#include "moses/InputPath.h"
	#include "moses/Range.h"
	#include "moses/StaticData.h"
	#include "moses/Hypothesis.h"
	#include "moses/Manager.h"
	#include "moses/FactorCollection.h"
	#include <cmath>

	using namespace std;

	namespace Moses
	{
	struct DistortionState : public FFState {
	Range range;
	int first_gap;
	bool inSubordinateConjunction;
	DistortionState(const Range& wr, int fg, bool subord=false) : range(wr), first_gap(fg), inSubordinateConjunction(subord) {}

	size_t hash() const {
	return range.GetEndPos();
	}
	virtual bool operator==(const FFState& other) const {
	const DistortionState& o =
	static_cast<const DistortionState&>(other);
	return ( (range.GetEndPos() == o.range.GetEndPos()) && (inSubordinateConjunction == o.inSubordinateConjunction) );
	}

	};

	std::vector<const DistortionScoreProducer*> DistortionScoreProducer::s_staticColl;

	DistortionScoreProducer::DistortionScoreProducer(const std::string &line)
	: StatefulFeatureFunction(1, line)
	, m_useSparse(false)
	, m_sparseDistance(false)
	, m_sparseSubordinate(false)
	{
	s_staticColl.push_back(this);
	ReadParameters();
	}

	void DistortionScoreProducer::SetParameter(const std::string& key, const std::string& value)
	{
	if (key == "sparse") {
	m_useSparse = Scan<bool>(value);
	} else if (key == "sparse-distance") {
	m_sparseDistance = Scan<bool>(value);
	} else if (key == "sparse-input-factor") {
	m_sparseFactorTypeSource = Scan<FactorType>(value);
	} else if (key == "sparse-output-factor") {
	m_sparseFactorTypeTarget = Scan<FactorType>(value);
	} else if (key == "sparse-subordinate") {
	std::string subordinateConjunctionTag = Scan<std::string>(value);
	FactorCollection &factorCollection = FactorCollection::Instance();
	m_subordinateConjunctionTagFactor = factorCollection.AddFactor(subordinateConjunctionTag,false);
	m_sparseSubordinate = true;
	} else if (key == "sparse-subordinate-output-factor") {
	m_sparseFactorTypeTargetSubordinate = Scan<FactorType>(value);
	} else {
	StatefulFeatureFunction::SetParameter(key, value);
	}
	}

	const FFState* DistortionScoreProducer::EmptyHypothesisState(const InputType &input) const
	{
	// fake previous translated phrase start and end
	size_t start = NOT_FOUND;
	size_t end = NOT_FOUND;
	if (input.m_frontSpanCoveredLength > 0) {
	// can happen with --continue-partial-translation
	start = 0;
	end = input.m_frontSpanCoveredLength -1;
	}
	return new DistortionState(
	Range(start, end),
	NOT_FOUND);
	}

	float
	DistortionScoreProducer::
	CalculateDistortionScore(const Hypothesis& hypo,
	const Range &prev, const Range &curr, const int FirstGap)
	{
	// if(!StaticData::Instance().UseEarlyDistortionCost()) {
	if(!hypo.GetManager().options()->reordering.use_early_distortion_cost) {
	return - (float) hypo.GetInput().ComputeDistortionDistance(prev, curr);
	} // else {

	/* Pay distortion score as soon as possible, from Moore and Quirk MT Summit 2007
	Definitions:
	S : current source range
	S' : last translated source phrase range
	S'' : longest fully-translated initial segment
	*/

	int prefixEndPos = (int)FirstGap-1;
	if((int)FirstGap==-1)
	prefixEndPos = -1;

	// case1: S is adjacent to S'' => return 0
	if ((int) curr.GetStartPos() == prefixEndPos+1) {
	IFVERBOSE(4) std::cerr<< "MQ07disto:case1" << std::endl;
	return 0;
	}

	// case2: S is to the left of S' => return 2(length(S))
	if ((int) curr.GetEndPos() < (int) prev.GetEndPos()) {
	IFVERBOSE(4) std::cerr<< "MQ07disto:case2" << std::endl;
	return (float) -2*(int)curr.GetNumWordsCovered();
	}

	// case3: S' is a subsequence of S'' => return 2(nbWordBetween(S,S'')+length(S))
	if ((int) prev.GetEndPos() <= prefixEndPos) {
	IFVERBOSE(4) std::cerr<< "MQ07disto:case3" << std::endl;
	int z = (int)curr.GetStartPos()-prefixEndPos - 1;
	return (float) -2*(z + (int)curr.GetNumWordsCovered());
	}

	// case4: otherwise => return 2(nbWordBetween(S,S')+length(S))
	IFVERBOSE(4) std::cerr<< "MQ07disto:case4" << std::endl;
	return (float) -2*((int)curr.GetNumWordsBetween(prev) + (int)curr.GetNumWordsCovered());

	}


	FFState* DistortionScoreProducer::EvaluateWhenApplied(
	const Hypothesis& hypo,
	const FFState* prev_state,
	ScoreComponentCollection* out) const
	{
	const DistortionState* prev = static_cast<const DistortionState*>(prev_state);
	bool subordinateConjunction = prev->inSubordinateConjunction;

	if (m_useSparse) {
	int jumpFromPos = prev->range.GetEndPos()+1;
	int jumpToPos = hypo.GetCurrSourceWordsRange().GetStartPos();
	size_t distance = std::abs( jumpFromPos - jumpToPos );

	const Sentence& sentence = static_cast<const Sentence&>(hypo.GetInput());

	StringPiece jumpFromSourceFactorPrev;
	StringPiece jumpFromSourceFactor;
	StringPiece jumpToSourceFactor;
	if (jumpFromPos < (int)sentence.GetSize()) {
	jumpFromSourceFactor = sentence.GetWord(jumpFromPos).GetFactor(m_sparseFactorTypeSource)->GetString();
	} else {
	jumpFromSourceFactor = "</s>";
	}
	if (jumpFromPos > 0) {
	jumpFromSourceFactorPrev = sentence.GetWord(jumpFromPos-1).GetFactor(m_sparseFactorTypeSource)->GetString();
	} else {
	jumpFromSourceFactorPrev = "<s>";
	}
	jumpToSourceFactor = sentence.GetWord(jumpToPos).GetFactor(m_sparseFactorTypeSource)->GetString();

	const TargetPhrase& currTargetPhrase = hypo.GetCurrTargetPhrase();
	StringPiece jumpToTargetFactor = currTargetPhrase.GetWord(0).GetFactor(m_sparseFactorTypeTarget)->GetString();

	util::StringStream featureName;

	// source factor (start position)
	featureName = util::StringStream();
	featureName << m_description << "_";
	if ( jumpToPos > jumpFromPos ) {
	featureName << "R";
	} else if ( jumpToPos < jumpFromPos ) {
	featureName << "L";
	} else {
	featureName << "M";
	}
	if (m_sparseDistance) {
	featureName << distance;
	}
	featureName << "_SFS_" << jumpFromSourceFactor;
	if (m_sparseSubordinate && subordinateConjunction) {
	featureName << "_SUBORD";
	}
	out->SparsePlusEquals(featureName.str(), 1);

	// source factor (start position minus 1)
	featureName = util::StringStream();
	featureName << m_description << "_";
	if ( jumpToPos > jumpFromPos ) {
	featureName << "R";
	} else if ( jumpToPos < jumpFromPos ) {
	featureName << "L";
	} else {
	featureName << "M";
	}
	if (m_sparseDistance) {
	featureName << distance;
	}
	featureName << "_SFP_" << jumpFromSourceFactorPrev;
	if (m_sparseSubordinate && subordinateConjunction) {
	featureName << "_SUBORD";
	}
	out->SparsePlusEquals(featureName.str(), 1);

	// source factor (end position)
	featureName = util::StringStream();
	featureName << m_description << "_";
	if ( jumpToPos > jumpFromPos ) {
	featureName << "R";
	} else if ( jumpToPos < jumpFromPos ) {
	featureName << "L";
	} else {
	featureName << "M";
	}
	if (m_sparseDistance) {
	featureName << distance;
	}
	featureName << "_SFE_" << jumpToSourceFactor;
	if (m_sparseSubordinate && subordinateConjunction) {
	featureName << "_SUBORD";
	}
	out->SparsePlusEquals(featureName.str(), 1);

	// target factor (end position)
	featureName = util::StringStream();
	featureName << m_description << "_";
	if ( jumpToPos > jumpFromPos ) {
	featureName << "R";
	} else if ( jumpToPos < jumpFromPos ) {
	featureName << "L";
	} else {
	featureName << "M";
	}
	if (m_sparseDistance) {
	featureName << distance;
	}
	featureName << "_TFE_" << jumpToTargetFactor;
	if (m_sparseSubordinate && subordinateConjunction) {
	featureName << "_SUBORD";
	}
	out->SparsePlusEquals(featureName.str(), 1);

	// relative source sentence position
	featureName = util::StringStream();
	featureName << m_description << "_";
	if ( jumpToPos > jumpFromPos ) {
	featureName << "R";
	} else if ( jumpToPos < jumpFromPos ) {
	featureName << "L";
	} else {
	featureName << "M";
	}
	if (m_sparseDistance) {
	featureName << distance;
	}
	size_t relativeSourceSentencePosBin = std::floor( 5 * (float)jumpFromPos / (sentence.GetSize()+1) );
	featureName << "_P_" << relativeSourceSentencePosBin;
	if (m_sparseSubordinate && subordinateConjunction) {
	featureName << "_SUBORD";
	}
	out->SparsePlusEquals(featureName.str(), 1);

	// source sentence length bin
	featureName = util::StringStream();
	featureName << m_description << "_";
	if ( jumpToPos > jumpFromPos ) {
	featureName << "R";
	} else if ( jumpToPos < jumpFromPos ) {
	featureName << "L";
	} else {
	featureName << "M";
	}
	if (m_sparseDistance) {
	featureName << distance;
	}
	size_t sourceSentenceLengthBin = 3;
	if (sentence.GetSize() < 15) {
	sourceSentenceLengthBin = 0;
	} else if (sentence.GetSize() < 23) {
	sourceSentenceLengthBin = 1;
	} else if (sentence.GetSize() < 33) {
	sourceSentenceLengthBin = 2;
	}
	featureName << "_SL_" << sourceSentenceLengthBin;
	if (m_sparseSubordinate && subordinateConjunction) {
	featureName << "_SUBORD";
	}
	out->SparsePlusEquals(featureName.str(), 1);

	if (m_sparseSubordinate) {
	for (size_t posT=0; posT<currTargetPhrase.GetSize(); ++posT) {
	const Word &wordT = currTargetPhrase.GetWord(posT);
	if (wordT[m_sparseFactorTypeTargetSubordinate] == m_subordinateConjunctionTagFactor) {
	subordinateConjunction = true;
	} else if (wordT[m_sparseFactorTypeTargetSubordinate]->GetString()[0] == 'V') {
	subordinateConjunction = false;
	}
	};
	}
	}

	const float distortionScore = CalculateDistortionScore(
	hypo,
	prev->range,
	hypo.GetCurrSourceWordsRange(),
	prev->first_gap);
	out->PlusEquals(this, distortionScore);

	DistortionState* state = new DistortionState(
	hypo.GetCurrSourceWordsRange(),
	hypo.GetWordsBitmap().GetFirstGapPos(),
	subordinateConjunction);

	return state;
	}


	}