Upload folder using huggingface_hub

41f6dd8 verified almost 2 years ago

4.58 kB

/***********************************************************************
  Moses - factored phrase-based language decoder
  Copyright (C) 2010 University of Edinburgh
  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.
  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.
  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 ***********************************************************************/

#include "SentenceAlignment.h"

#include <map>
#include <set>
#include <string>

#include "tables-core.h"
#include "util/tokenize.hh"

using namespace std;

namespace MosesTraining
{

SentenceAlignment::~SentenceAlignment() {}

void addBoundaryWords(vector<string> &phrase)
{
  phrase.insert(phrase.begin(), "<s>");
  phrase.push_back("</s>");
}

bool SentenceAlignment::processTargetSentence(const char * targetString, int, bool boundaryRules)
{
  target = util::tokenize(targetString);
  if (boundaryRules)
    addBoundaryWords(target);
  return true;
}

bool SentenceAlignment::processSourceSentence(const char * sourceString, int, bool boundaryRules)
{
  source = util::tokenize(sourceString);
  if (boundaryRules)
    addBoundaryWords(source);
  return true;
}

bool SentenceAlignment::create(const char targetString[],
                               const char sourceString[],
                               const char alignmentString[],
                               const char weightString[],
                               int sentenceID, bool boundaryRules)
{
  using namespace std;
  this->sentenceID = sentenceID;
  this->weightString = std::string(weightString);

  // process sentence strings and store in target and source members.
  if (!processTargetSentence(targetString, sentenceID, boundaryRules)) {
    return false;
  }
  if (!processSourceSentence(sourceString, sentenceID, boundaryRules)) {
    return false;
  }

  // check if sentences are empty
  if (target.size() == 0 || source.size() == 0) {
    cerr << "no target (" << target.size() << ") or source (" << source.size() << ") words << end insentence " << sentenceID << endl;
    cerr << "T: " << targetString << endl << "S: " << sourceString << endl;
    return false;
  }

  // prepare data structures for alignments
  for(size_t i=0; i<source.size(); i++) {
    alignedCountS.push_back( 0 );
  }
  for(size_t i=0; i<target.size(); i++) {
    vector< int > dummy;
    alignedToT.push_back( dummy );
  }

  // reading in alignments
  vector<string> alignmentSequence = util::tokenize( alignmentString );
  for(size_t i=0; i<alignmentSequence.size(); i++) {
    int s,t;
    // cout << "scaning " << alignmentSequence[i].c_str() << endl;
    if (! sscanf(alignmentSequence[i].c_str(), "%d-%d", &s, &t)) {
      cerr << "WARNING: " << alignmentSequence[i] << " is a bad alignment point in sentence " << sentenceID << endl;
      cerr << "T: " << targetString << endl << "S: " << sourceString << endl;
      return false;
    }

    if (boundaryRules) {
      ++s;
      ++t;
    }

    // cout << "alignmentSequence[i] " << alignmentSequence[i] << " is " << s << ", " << t << endl;
    if ((size_t)t >= target.size() || (size_t)s >= source.size()) {
      cerr << "WARNING: sentence " << sentenceID << " has alignment point (" << s << ", " << t << ") out of bounds (" << source.size() << ", " << target.size() << ")\n";
      cerr << "T: " << targetString << endl << "S: " << sourceString << endl;
      return false;
    }
    alignedToT[t].push_back( s );
    alignedCountS[s]++;
  }

  if (boundaryRules) {
    alignedToT[0].push_back(0);
    alignedCountS[0]++;

    alignedToT.back().push_back(alignedCountS.size() - 1);
    alignedCountS.back()++;

  }

  return true;
}

void SentenceAlignment::invertAlignment()
{
  alignedToS.resize(source.size());
  for (size_t targetPos = 0; targetPos < alignedToT.size(); ++targetPos) {
    const std::vector<int> &vec = alignedToT[targetPos];
    for (size_t i = 0; i < vec.size(); ++i) {
      int sourcePos = vec[i];
      alignedToS[sourcePos].push_back(targetPos);
    }

  }
}

}

	/***********************************************************************
	Moses - factored phrase-based language decoder
	Copyright (C) 2010 University of Edinburgh

	This library is free software; you can redistribute it and/or
	modify it under the terms of the GNU Lesser General Public
	License as published by the Free Software Foundation; either
	version 2.1 of the License, or (at your option) any later version.

	This library is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	Lesser General Public License for more details.

	You should have received a copy of the GNU Lesser General Public
	License along with this library; if not, write to the Free Software
	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	***********************************************************************/

	#include "SentenceAlignment.h"

	#include <map>
	#include <set>
	#include <string>

	#include "tables-core.h"
	#include "util/tokenize.hh"

	using namespace std;

	namespace MosesTraining
	{

	SentenceAlignment::~SentenceAlignment() {}

	void addBoundaryWords(vector<string> &phrase)
	{
	phrase.insert(phrase.begin(), "<s>");
	phrase.push_back("</s>");
	}

	bool SentenceAlignment::processTargetSentence(const char * targetString, int, bool boundaryRules)
	{
	target = util::tokenize(targetString);
	if (boundaryRules)
	addBoundaryWords(target);
	return true;
	}

	bool SentenceAlignment::processSourceSentence(const char * sourceString, int, bool boundaryRules)
	{
	source = util::tokenize(sourceString);
	if (boundaryRules)
	addBoundaryWords(source);
	return true;
	}

	bool SentenceAlignment::create(const char targetString[],
	const char sourceString[],
	const char alignmentString[],
	const char weightString[],
	int sentenceID, bool boundaryRules)
	{
	using namespace std;
	this->sentenceID = sentenceID;
	this->weightString = std::string(weightString);

	// process sentence strings and store in target and source members.
	if (!processTargetSentence(targetString, sentenceID, boundaryRules)) {
	return false;
	}
	if (!processSourceSentence(sourceString, sentenceID, boundaryRules)) {
	return false;
	}

	// check if sentences are empty
	if (target.size() == 0 \|\| source.size() == 0) {
	cerr << "no target (" << target.size() << ") or source (" << source.size() << ") words << end insentence " << sentenceID << endl;
	cerr << "T: " << targetString << endl << "S: " << sourceString << endl;
	return false;
	}

	// prepare data structures for alignments
	for(size_t i=0; i<source.size(); i++) {
	alignedCountS.push_back( 0 );
	}
	for(size_t i=0; i<target.size(); i++) {
	vector< int > dummy;
	alignedToT.push_back( dummy );
	}

	// reading in alignments
	vector<string> alignmentSequence = util::tokenize( alignmentString );
	for(size_t i=0; i<alignmentSequence.size(); i++) {
	int s,t;
	// cout << "scaning " << alignmentSequence[i].c_str() << endl;
	if (! sscanf(alignmentSequence[i].c_str(), "%d-%d", &s, &t)) {
	cerr << "WARNING: " << alignmentSequence[i] << " is a bad alignment point in sentence " << sentenceID << endl;
	cerr << "T: " << targetString << endl << "S: " << sourceString << endl;
	return false;
	}

	if (boundaryRules) {
	++s;
	++t;
	}

	// cout << "alignmentSequence[i] " << alignmentSequence[i] << " is " << s << ", " << t << endl;
	if ((size_t)t >= target.size() \|\| (size_t)s >= source.size()) {
	cerr << "WARNING: sentence " << sentenceID << " has alignment point (" << s << ", " << t << ") out of bounds (" << source.size() << ", " << target.size() << ")\n";
	cerr << "T: " << targetString << endl << "S: " << sourceString << endl;
	return false;
	}
	alignedToT[t].push_back( s );
	alignedCountS[s]++;
	}

	if (boundaryRules) {
	alignedToT[0].push_back(0);
	alignedCountS[0]++;

	alignedToT.back().push_back(alignedCountS.size() - 1);
	alignedCountS.back()++;

	}

	return true;
	}

	void SentenceAlignment::invertAlignment()
	{
	alignedToS.resize(source.size());
	for (size_t targetPos = 0; targetPos < alignedToT.size(); ++targetPos) {
	const std::vector<int> &vec = alignedToT[targetPos];
	for (size_t i = 0; i < vec.size(); ++i) {
	int sourcePos = vec[i];
	alignedToS[sourcePos].push_back(targetPos);
	}

	}
	}

	}