wiki-cased / tools /mosesdecoder /lm /partial_test.cc

Upload folder using huggingface_hub

41f6dd8 verified almost 2 years ago

6.68 kB

#include "lm/partial.hh"

#include "lm/left.hh"
#include "lm/model.hh"
#include "util/tokenize_piece.hh"

#define BOOST_TEST_MODULE PartialTest
#include <boost/test/unit_test.hpp>
#include <boost/test/floating_point_comparison.hpp>

namespace lm {
namespace ngram {
namespace {

const char *TestLocation() {
  if (boost::unit_test::framework::master_test_suite().argc < 2) {
    return "test.arpa";
  }
  return boost::unit_test::framework::master_test_suite().argv[1];
}

Config SilentConfig() {
  Config config;
  config.arpa_complain = Config::NONE;
  config.messages = NULL;
  return config;
}

struct ModelFixture {
  ModelFixture() : m(TestLocation(), SilentConfig()) {}

  RestProbingModel m;
};

BOOST_FIXTURE_TEST_SUITE(suite, ModelFixture)

BOOST_AUTO_TEST_CASE(SimpleBefore) {
  Left left;
  left.full = false;
  left.length = 0;
  Right right;
  right.length = 0;

  Right reveal;
  reveal.length = 1;
  WordIndex period = m.GetVocabulary().Index(".");
  reveal.words[0] = period;
  reveal.backoff[0] = -0.845098;

  BOOST_CHECK_CLOSE(0.0, RevealBefore(m, reveal, 0, false, left, right), 0.001);
  BOOST_CHECK_EQUAL(0, left.length);
  BOOST_CHECK(!left.full);
  BOOST_CHECK_EQUAL(1, right.length);
  BOOST_CHECK_EQUAL(period, right.words[0]);
  BOOST_CHECK_CLOSE(-0.845098, right.backoff[0], 0.001);

  WordIndex more = m.GetVocabulary().Index("more");
  reveal.words[1] = more;
  reveal.backoff[1] =  -0.4771212;
  reveal.length = 2;
  BOOST_CHECK_CLOSE(0.0, RevealBefore(m, reveal, 1, false, left, right), 0.001);
  BOOST_CHECK_EQUAL(0, left.length);
  BOOST_CHECK(!left.full);
  BOOST_CHECK_EQUAL(2, right.length);
  BOOST_CHECK_EQUAL(period, right.words[0]);
  BOOST_CHECK_EQUAL(more, right.words[1]);
  BOOST_CHECK_CLOSE(-0.845098, right.backoff[0], 0.001);
  BOOST_CHECK_CLOSE(-0.4771212, right.backoff[1], 0.001);
}

BOOST_AUTO_TEST_CASE(AlsoWouldConsider) {
  WordIndex would = m.GetVocabulary().Index("would");
  WordIndex consider = m.GetVocabulary().Index("consider");

  ChartState current;
  current.left.length = 1;
  current.left.pointers[0] = would;
  current.left.full = false;
  current.right.length = 1;
  current.right.words[0] = would;
  current.right.backoff[0] = -0.30103;

  Left after;
  after.full = false;
  after.length = 1;
  after.pointers[0] = consider;

  // adjustment for would consider
  BOOST_CHECK_CLOSE(-1.687872 - -0.2922095 - 0.30103, RevealAfter(m, current.left, current.right, after, 0), 0.001);

  BOOST_CHECK_EQUAL(2, current.left.length);
  BOOST_CHECK_EQUAL(would, current.left.pointers[0]);
  BOOST_CHECK_EQUAL(false, current.left.full);

  WordIndex also = m.GetVocabulary().Index("also");
  Right before;
  before.length = 1;
  before.words[0] = also;
  before.backoff[0] = -0.30103;
  // r(would) = -0.2922095 [i would], r(would -> consider) = -1.988902 [b(would) + p(consider)]
  // p(also -> would) = -2, p(also would -> consider) = -3
  BOOST_CHECK_CLOSE(-2 + 0.2922095 -3 + 1.988902, RevealBefore(m, before, 0, false, current.left, current.right), 0.001);
  BOOST_CHECK_EQUAL(0, current.left.length);
  BOOST_CHECK(current.left.full);
  BOOST_CHECK_EQUAL(2, current.right.length);
  BOOST_CHECK_EQUAL(would, current.right.words[0]);
  BOOST_CHECK_EQUAL(also, current.right.words[1]);
}

BOOST_AUTO_TEST_CASE(EndSentence) {
  WordIndex loin = m.GetVocabulary().Index("loin");
  WordIndex period = m.GetVocabulary().Index(".");
  WordIndex eos = m.GetVocabulary().EndSentence();

  ChartState between;
  between.left.length = 1;
  between.left.pointers[0] = eos;
  between.left.full = true;
  between.right.length = 0;

  Right before;
  before.words[0] = period;
  before.words[1] = loin;
  before.backoff[0] = -0.845098;
  before.backoff[1] = 0.0;

  before.length = 1;
  BOOST_CHECK_CLOSE(-0.0410707, RevealBefore(m, before, 0, true, between.left, between.right), 0.001);
  BOOST_CHECK_EQUAL(0, between.left.length);
}

float ScoreFragment(const RestProbingModel &model, unsigned int *begin, unsigned int *end, ChartState &out) {
  RuleScore<RestProbingModel> scorer(model, out);
  for (unsigned int *i = begin; i < end; ++i) {
    scorer.Terminal(*i);
  }
  return scorer.Finish();
}

void CheckAdjustment(const RestProbingModel &model, float expect, const Right &before_in, bool before_full, ChartState between, const Left &after_in) {
  Right before(before_in);
  Left after(after_in);
  after.full = false;
  float got = 0.0;
  for (unsigned int i = 1; i < 5; ++i) {
    if (before_in.length >= i) {
      before.length = i;
      got += RevealBefore(model, before, i - 1, false, between.left, between.right);
    }
    if (after_in.length >= i) {
      after.length = i;
      got += RevealAfter(model, between.left, between.right, after, i - 1);
    }
  }
  if (after_in.full) {
    after.full = true;
    got += RevealAfter(model, between.left, between.right, after, after.length);
  }
  if (before_full) {
    got += RevealBefore(model, before, before.length, true, between.left, between.right);
  }
  // Sometimes they're zero and BOOST_CHECK_CLOSE fails for this.
  BOOST_CHECK(fabs(expect - got) < 0.001);
}

void FullDivide(const RestProbingModel &model, StringPiece str) {
  std::vector<WordIndex> indices;
  for (util::TokenIter<util::SingleCharacter, true> i(str, ' '); i; ++i) {
    indices.push_back(model.GetVocabulary().Index(*i));
  }
  ChartState full_state;
  float full = ScoreFragment(model, &indices.front(), &indices.back() + 1, full_state);

  ChartState before_state;
  before_state.left.full = false;
  RuleScore<RestProbingModel> before_scorer(model, before_state);
  float before_score = 0.0;
  for (unsigned int before = 0; before < indices.size(); ++before) {
    for (unsigned int after = before; after <= indices.size(); ++after) {
      ChartState after_state, between_state;
      float after_score = ScoreFragment(model, &indices.front() + after, &indices.front() + indices.size(), after_state);
      float between_score = ScoreFragment(model, &indices.front() + before, &indices.front() + after, between_state);
      CheckAdjustment(model, full - before_score - after_score - between_score, before_state.right, before_state.left.full, between_state, after_state.left);
    }
    before_scorer.Terminal(indices[before]);
    before_score = before_scorer.Finish();
  }
}

BOOST_AUTO_TEST_CASE(Strings) {
  FullDivide(m, "also would consider");
  FullDivide(m, "looking on a little more loin . </s>");
  FullDivide(m, "in biarritz watching considering looking . on a little more loin also would consider higher to look good unknown the screening foo bar , unknown however unknown </s>");
}

BOOST_AUTO_TEST_SUITE_END()
} // namespace
} // namespace ngram
} // namespace lm

	#include "lm/partial.hh"

	#include "lm/left.hh"
	#include "lm/model.hh"
	#include "util/tokenize_piece.hh"

	#define BOOST_TEST_MODULE PartialTest
	#include <boost/test/unit_test.hpp>
	#include <boost/test/floating_point_comparison.hpp>

	namespace lm {
	namespace ngram {
	namespace {

	const char *TestLocation() {
	if (boost::unit_test::framework::master_test_suite().argc < 2) {
	return "test.arpa";
	}
	return boost::unit_test::framework::master_test_suite().argv[1];
	}

	Config SilentConfig() {
	Config config;
	config.arpa_complain = Config::NONE;
	config.messages = NULL;
	return config;
	}

	struct ModelFixture {
	ModelFixture() : m(TestLocation(), SilentConfig()) {}

	RestProbingModel m;
	};

	BOOST_FIXTURE_TEST_SUITE(suite, ModelFixture)

	BOOST_AUTO_TEST_CASE(SimpleBefore) {
	Left left;
	left.full = false;
	left.length = 0;
	Right right;
	right.length = 0;

	Right reveal;
	reveal.length = 1;
	WordIndex period = m.GetVocabulary().Index(".");
	reveal.words[0] = period;
	reveal.backoff[0] = -0.845098;

	BOOST_CHECK_CLOSE(0.0, RevealBefore(m, reveal, 0, false, left, right), 0.001);
	BOOST_CHECK_EQUAL(0, left.length);
	BOOST_CHECK(!left.full);
	BOOST_CHECK_EQUAL(1, right.length);
	BOOST_CHECK_EQUAL(period, right.words[0]);
	BOOST_CHECK_CLOSE(-0.845098, right.backoff[0], 0.001);

	WordIndex more = m.GetVocabulary().Index("more");
	reveal.words[1] = more;
	reveal.backoff[1] = -0.4771212;
	reveal.length = 2;
	BOOST_CHECK_CLOSE(0.0, RevealBefore(m, reveal, 1, false, left, right), 0.001);
	BOOST_CHECK_EQUAL(0, left.length);
	BOOST_CHECK(!left.full);
	BOOST_CHECK_EQUAL(2, right.length);
	BOOST_CHECK_EQUAL(period, right.words[0]);
	BOOST_CHECK_EQUAL(more, right.words[1]);
	BOOST_CHECK_CLOSE(-0.845098, right.backoff[0], 0.001);
	BOOST_CHECK_CLOSE(-0.4771212, right.backoff[1], 0.001);
	}

	BOOST_AUTO_TEST_CASE(AlsoWouldConsider) {
	WordIndex would = m.GetVocabulary().Index("would");
	WordIndex consider = m.GetVocabulary().Index("consider");

	ChartState current;
	current.left.length = 1;
	current.left.pointers[0] = would;
	current.left.full = false;
	current.right.length = 1;
	current.right.words[0] = would;
	current.right.backoff[0] = -0.30103;

	Left after;
	after.full = false;
	after.length = 1;
	after.pointers[0] = consider;

	// adjustment for would consider
	BOOST_CHECK_CLOSE(-1.687872 - -0.2922095 - 0.30103, RevealAfter(m, current.left, current.right, after, 0), 0.001);

	BOOST_CHECK_EQUAL(2, current.left.length);
	BOOST_CHECK_EQUAL(would, current.left.pointers[0]);
	BOOST_CHECK_EQUAL(false, current.left.full);

	WordIndex also = m.GetVocabulary().Index("also");
	Right before;
	before.length = 1;
	before.words[0] = also;
	before.backoff[0] = -0.30103;
	// r(would) = -0.2922095 [i would], r(would -> consider) = -1.988902 [b(would) + p(consider)]
	// p(also -> would) = -2, p(also would -> consider) = -3
	BOOST_CHECK_CLOSE(-2 + 0.2922095 -3 + 1.988902, RevealBefore(m, before, 0, false, current.left, current.right), 0.001);
	BOOST_CHECK_EQUAL(0, current.left.length);
	BOOST_CHECK(current.left.full);
	BOOST_CHECK_EQUAL(2, current.right.length);
	BOOST_CHECK_EQUAL(would, current.right.words[0]);
	BOOST_CHECK_EQUAL(also, current.right.words[1]);
	}

	BOOST_AUTO_TEST_CASE(EndSentence) {
	WordIndex loin = m.GetVocabulary().Index("loin");
	WordIndex period = m.GetVocabulary().Index(".");
	WordIndex eos = m.GetVocabulary().EndSentence();

	ChartState between;
	between.left.length = 1;
	between.left.pointers[0] = eos;
	between.left.full = true;
	between.right.length = 0;

	Right before;
	before.words[0] = period;
	before.words[1] = loin;
	before.backoff[0] = -0.845098;
	before.backoff[1] = 0.0;

	before.length = 1;
	BOOST_CHECK_CLOSE(-0.0410707, RevealBefore(m, before, 0, true, between.left, between.right), 0.001);
	BOOST_CHECK_EQUAL(0, between.left.length);
	}

	float ScoreFragment(const RestProbingModel &model, unsigned int begin, unsigned int end, ChartState &out) {
	RuleScore<RestProbingModel> scorer(model, out);
	for (unsigned int *i = begin; i < end; ++i) {
	scorer.Terminal(*i);
	}
	return scorer.Finish();
	}

	void CheckAdjustment(const RestProbingModel &model, float expect, const Right &before_in, bool before_full, ChartState between, const Left &after_in) {
	Right before(before_in);
	Left after(after_in);
	after.full = false;
	float got = 0.0;
	for (unsigned int i = 1; i < 5; ++i) {
	if (before_in.length >= i) {
	before.length = i;
	got += RevealBefore(model, before, i - 1, false, between.left, between.right);
	}
	if (after_in.length >= i) {
	after.length = i;
	got += RevealAfter(model, between.left, between.right, after, i - 1);
	}
	}
	if (after_in.full) {
	after.full = true;
	got += RevealAfter(model, between.left, between.right, after, after.length);
	}
	if (before_full) {
	got += RevealBefore(model, before, before.length, true, between.left, between.right);
	}
	// Sometimes they're zero and BOOST_CHECK_CLOSE fails for this.
	BOOST_CHECK(fabs(expect - got) < 0.001);
	}

	void FullDivide(const RestProbingModel &model, StringPiece str) {
	std::vector<WordIndex> indices;
	for (util::TokenIter<util::SingleCharacter, true> i(str, ' '); i; ++i) {
	indices.push_back(model.GetVocabulary().Index(*i));
	}
	ChartState full_state;
	float full = ScoreFragment(model, &indices.front(), &indices.back() + 1, full_state);

	ChartState before_state;
	before_state.left.full = false;
	RuleScore<RestProbingModel> before_scorer(model, before_state);
	float before_score = 0.0;
	for (unsigned int before = 0; before < indices.size(); ++before) {
	for (unsigned int after = before; after <= indices.size(); ++after) {
	ChartState after_state, between_state;
	float after_score = ScoreFragment(model, &indices.front() + after, &indices.front() + indices.size(), after_state);
	float between_score = ScoreFragment(model, &indices.front() + before, &indices.front() + after, between_state);
	CheckAdjustment(model, full - before_score - after_score - between_score, before_state.right, before_state.left.full, between_state, after_state.left);
	}
	before_scorer.Terminal(indices[before]);
	before_score = before_scorer.Finish();
	}
	}

	BOOST_AUTO_TEST_CASE(Strings) {
	FullDivide(m, "also would consider");
	FullDivide(m, "looking on a little more loin . </s>");
	FullDivide(m, "in biarritz watching considering looking . on a little more loin also would consider higher to look good unknown the screening foo bar , unknown however unknown </s>");
	}

	BOOST_AUTO_TEST_SUITE_END()
	} // namespace
	} // namespace ngram
	} // namespace lm