wiki-cased / tools /mosesdecoder /phrase-extract /PropertiesConsolidator.cpp

Upload folder using huggingface_hub

41f6dd8 verified almost 2 years ago

12.4 kB

/***********************************************************************
  Moses - factored phrase-based language decoder
  Copyright (C) University of Edinburgh
  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.
  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.
  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 ***********************************************************************/

#include "PropertiesConsolidator.h"

#include <sstream>
#include <limits>
#include <vector>

#include "moses/Util.h"
#include "phrase-extract/InputFileStream.h"
#include "phrase-extract/OutputFileStream.h"


namespace MosesTraining
{

void PropertiesConsolidator::ActivateSourceLabelsProcessing(const std::string &sourceLabelSetFile)
{
  Moses::InputFileStream inFile(sourceLabelSetFile);

  // read source label set
  m_sourceLabels.clear();
  std::string line;
  while (getline(inFile, line)) {
    std::istringstream tokenizer(line);
    std::string label;
    size_t index;
    try {
      tokenizer >> label >> index;
    } catch (const std::exception &e) {
      UTIL_THROW2("Error reading source label set file " << sourceLabelSetFile << " .");
    }
    std::pair< std::map<std::string,size_t>::iterator, bool > inserted = m_sourceLabels.insert( std::pair<std::string,size_t>(label,index) );
    UTIL_THROW_IF2(!inserted.second,"Source label set file " << sourceLabelSetFile << " should contain each syntactic label only once.");
  }

  inFile.Close();

  m_sourceLabelsFlag = true;
}


void PropertiesConsolidator::ActivatePartsOfSpeechProcessing(const std::string &partsOfSpeechFile)
{
  Moses::InputFileStream inFile(partsOfSpeechFile);

  // read parts-of-speech vocabulary
  m_partsOfSpeechVocabulary.clear();
  std::string line;
  while (getline(inFile, line)) {
    std::istringstream tokenizer(line);
    std::string label;
    size_t index;
    try {
      tokenizer >> label >> index;
    } catch (const std::exception &e) {
      UTIL_THROW2("Error reading part-of-speech vocabulary file " << partsOfSpeechFile << " .");
    }
    std::pair< std::map<std::string,size_t>::iterator, bool > inserted = m_partsOfSpeechVocabulary.insert( std::pair<std::string,size_t>(label,index) );
    UTIL_THROW_IF2(!inserted.second,"Part-of-speech vocabulary file " << partsOfSpeechFile << " should contain each POS tag only once.");
  }

  inFile.Close();

  m_partsOfSpeechFlag = true;
}


void PropertiesConsolidator::ActivateTargetSyntacticPreferencesProcessing(const std::string &targetSyntacticPreferencesLabelSetFile)
{
  Moses::InputFileStream inFile(targetSyntacticPreferencesLabelSetFile);

  // read target syntactic preferences label set
  m_targetSyntacticPreferencesLabels.clear();
  std::string line;
  while (getline(inFile, line)) {
    std::istringstream tokenizer(line);
    std::string label;
    size_t index;
    try {
      tokenizer >> label >> index;
    } catch (const std::exception &e) {
      UTIL_THROW2("Error reading target syntactic preferences label set file " << targetSyntacticPreferencesLabelSetFile << " .");
    }
    std::pair< std::map<std::string,size_t>::iterator, bool > inserted = m_targetSyntacticPreferencesLabels.insert( std::pair<std::string,size_t>(label,index) );
    UTIL_THROW_IF2(!inserted.second,"Target syntactic preferences label set file " << targetSyntacticPreferencesLabelSetFile << " should contain each syntactic label only once.");
  }

  inFile.Close();

  m_targetSyntacticPreferencesFlag = true;
}


void PropertiesConsolidator::ProcessPropertiesString(const std::string &propertiesString, Moses::OutputFileStream& out) const
{
  if ( propertiesString.empty() ) {
    return;
  }

  std::vector<std::string> toks;
  Moses::TokenizeMultiCharSeparator(toks, propertiesString, "{{");
  for (size_t i = 1; i < toks.size(); ++i) {
    std::string &tok = toks[i];
    if (tok.empty()) {
      continue;
    }
    size_t endPos = tok.rfind("}");
    tok = tok.substr(0, endPos - 1);
    std::vector<std::string> keyValue = Moses::TokenizeFirstOnly(tok, " ");
    assert(keyValue.size() == 2);

    if ( !keyValue[0].compare("SourceLabels") ) {

      if ( m_sourceLabelsFlag ) {

        // SourceLabels property: replace strings with vocabulary indices
        out << " {{" << keyValue[0];
        ProcessSourceLabelsPropertyValue(keyValue[1], out);
        out << "}}";

      } else { // don't process SourceLabels property
        out << " {{" << keyValue[0] << " " << keyValue[1] << "}}";
      }

    } else if ( !keyValue[0].compare("POS") ) {

      /* DO NOTHING (property is not registered in the decoder at the moment)
            if ( m_partsOfSpeechFlag ) {
              // POS property: replace strings with vocabulary indices
              out << " {{" << keyValue[0];
              ProcessPOSPropertyValue(keyValue[1], out);
              out << "}}";
            } else { // don't process POS property
              out << " {{" << keyValue[0] << " " << keyValue[1] << "}}";
            }
      */

    } else if ( !keyValue[0].compare("TargetPreferences") ) {

      if ( m_targetSyntacticPreferencesFlag ) {

        // TargetPreferences property: replace strings with vocabulary indices
        out << " {{" << keyValue[0];
        ProcessTargetSyntacticPreferencesPropertyValue(keyValue[1], out);
        out << "}}";

      } else { // don't process TargetPreferences property
        out << " {{" << keyValue[0] << " " << keyValue[1] << "}}";
      }

    } else {

      // output other property
      out << " {{" << keyValue[0] << " " << keyValue[1] << "}}";
    }
  }
}


void PropertiesConsolidator::ProcessSourceLabelsPropertyValue(const std::string &value, Moses::OutputFileStream& out) const
{
  // SourceLabels property: replace strings with vocabulary indices
  std::istringstream tokenizer(value);

  size_t nNTs;
  double totalCount;

  if (! (tokenizer >> nNTs)) { // first token: number of non-terminals (incl. left-hand side)
    UTIL_THROW2("Not able to read number of non-terminals from SourceLabels property. "
                << "Flawed SourceLabels property?");
  }
  assert( nNTs > 0 );
  out << " " << nNTs;

  if (! (tokenizer >> totalCount)) { // second token: overall rule count
    UTIL_THROW2("Not able to read overall rule count from SourceLabels property. "
                << "Flawed SourceLabels property?");
  }
  assert( totalCount > 0.0 );
  out << " " << totalCount;

  while (tokenizer.peek() != EOF) {
    try {

      size_t numberOfLHSsGivenRHS = std::numeric_limits<std::size_t>::max();

      std::string token;

      if (nNTs > 1) { // rule has right-hand side non-terminals, i.e. it's a hierarchical rule
        for (size_t i=0; i<nNTs-1; ++i) { // RHS source non-terminal labels
          tokenizer >> token; // RHS source non-terminal label
          std::map<std::string,size_t>::const_iterator found = m_sourceLabels.find(token);
          UTIL_THROW_IF2(found == m_sourceLabels.end(), "Label \"" << token << "\" from the phrase table not found in given label set.");
          out << " " << found->second;
        }

        tokenizer >> token; // sourceLabelsRHSCount
        out << " " << token;

        tokenizer >> numberOfLHSsGivenRHS;
        out << " " << numberOfLHSsGivenRHS;
      }

      for (size_t i=0; i<numberOfLHSsGivenRHS && tokenizer.peek()!=EOF; ++i) { // LHS source non-terminal labels seen with this RHS
        tokenizer >> token; // LHS source non-terminal label
        std::map<std::string,size_t>::const_iterator found = m_sourceLabels.find(token);
        UTIL_THROW_IF2(found == m_sourceLabels.end() ,"Label \"" << token << "\" from the phrase table not found in given label set.");
        out << " " << found->second;

        tokenizer >> token; // ruleSourceLabelledCount
        out << " " << token;
      }

    } catch (const std::exception &e) {
      UTIL_THROW2("Flawed item in SourceLabels property?");
    }
  }
}


void PropertiesConsolidator::ProcessPOSPropertyValue(const std::string &value, Moses::OutputFileStream& out) const
{
  std::istringstream tokenizer(value);
  while (tokenizer.peek() != EOF) {
    std::string token;
    tokenizer >> token;
    std::map<std::string,size_t>::const_iterator found = m_partsOfSpeechVocabulary.find(token);
    UTIL_THROW_IF2(found == m_partsOfSpeechVocabulary.end() ,"Part-of-speech \"" << token << "\" from the phrase table not found in given part-of-speech vocabulary.");
    out << " " << found->second;
  }
}


bool PropertiesConsolidator::GetPOSPropertyValueFromPropertiesString(const std::string &propertiesString, std::vector<std::string>& out) const
{
  out.clear();
  if ( propertiesString.empty() ) {
    return false;
  }

  std::vector<std::string> toks;
  Moses::TokenizeMultiCharSeparator(toks, propertiesString, "{{");
  for (size_t i = 1; i < toks.size(); ++i) {
    std::string &tok = toks[i];
    if (tok.empty()) {
      continue;
    }
    size_t endPos = tok.rfind("}");
    tok = tok.substr(0, endPos - 1);
    std::vector<std::string> keyValue = Moses::TokenizeFirstOnly(tok, " ");
    assert(keyValue.size() == 2);

    if ( !keyValue[0].compare("POS") ) {
      std::istringstream tokenizer(keyValue[1]);
      while (tokenizer.peek() != EOF) {
        std::string token;
        tokenizer >> token;
        out.push_back(token);
      }
      return true;
    }
  }

  return false;
}


void PropertiesConsolidator::ProcessTargetSyntacticPreferencesPropertyValue(const std::string &value, Moses::OutputFileStream& out) const
{
  // TargetPreferences property: replace strings with vocabulary indices
  std::istringstream tokenizer(value);

  size_t nNTs;
  double totalCount;

  if (! (tokenizer >> nNTs)) { // first token: number of non-terminals (incl. left-hand side)
    UTIL_THROW2("Not able to read number of non-terminals from TargetPreferences property. "
                << "Flawed TargetPreferences property?");
  }
  assert( nNTs > 0 );
  out << " " << nNTs;

  if (! (tokenizer >> totalCount)) { // second token: overall rule count
    UTIL_THROW2("Not able to read overall rule count from TargetPreferences property. "
                << "Flawed TargetPreferences property?");
  }
  assert( totalCount > 0.0 );
  out << " " << totalCount;

  while (tokenizer.peek() != EOF) {
    try {

      size_t numberOfLHSsGivenRHS = std::numeric_limits<std::size_t>::max();

      std::string token;

      if (nNTs > 1) { // rule has right-hand side non-terminals, i.e. it's a hierarchical rule
        for (size_t i=0; i<nNTs-1; ++i) { // RHS target preference non-terminal labels
          tokenizer >> token; // RHS target preference non-terminal label
          std::map<std::string,size_t>::const_iterator found = m_targetSyntacticPreferencesLabels.find(token);
          UTIL_THROW_IF2(found == m_targetSyntacticPreferencesLabels.end(), "Label \"" << token << "\" from the phrase table not found in given label set.");
          out << " " << found->second;
        }

        tokenizer >> token; // targetPreferenceRHSCount
        out << " " << token;

        tokenizer >> numberOfLHSsGivenRHS;
        out << " " << numberOfLHSsGivenRHS;
      }

      for (size_t i=0; i<numberOfLHSsGivenRHS && tokenizer.peek()!=EOF; ++i) { // LHS target preference non-terminal labels seen with this RHS
        tokenizer >> token; // LHS target preference non-terminal label
        std::map<std::string,size_t>::const_iterator found = m_targetSyntacticPreferencesLabels.find(token);
        UTIL_THROW_IF2(found == m_targetSyntacticPreferencesLabels.end() ,"Label \"" << token << "\" from the phrase table not found in given label set.");
        out << " " << found->second;

        tokenizer >> token; // ruleTargetPreferenceLabelledCount
        out << " " << token;
      }

    } catch (const std::exception &e) {
      UTIL_THROW2("Flawed item in TargetPreferences property?");
    }
  }
}


}  // namespace MosesTraining

	/***********************************************************************
	Moses - factored phrase-based language decoder
	Copyright (C) University of Edinburgh

	This library is free software; you can redistribute it and/or
	modify it under the terms of the GNU Lesser General Public
	License as published by the Free Software Foundation; either
	version 2.1 of the License, or (at your option) any later version.

	This library is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	Lesser General Public License for more details.

	You should have received a copy of the GNU Lesser General Public
	License along with this library; if not, write to the Free Software
	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	***********************************************************************/

	#include "PropertiesConsolidator.h"

	#include <sstream>
	#include <limits>
	#include <vector>

	#include "moses/Util.h"
	#include "phrase-extract/InputFileStream.h"
	#include "phrase-extract/OutputFileStream.h"


	namespace MosesTraining
	{

	void PropertiesConsolidator::ActivateSourceLabelsProcessing(const std::string &sourceLabelSetFile)
	{
	Moses::InputFileStream inFile(sourceLabelSetFile);

	// read source label set
	m_sourceLabels.clear();
	std::string line;
	while (getline(inFile, line)) {
	std::istringstream tokenizer(line);
	std::string label;
	size_t index;
	try {
	tokenizer >> label >> index;
	} catch (const std::exception &e) {
	UTIL_THROW2("Error reading source label set file " << sourceLabelSetFile << " .");
	}
	std::pair< std::map<std::string,size_t>::iterator, bool > inserted = m_sourceLabels.insert( std::pair<std::string,size_t>(label,index) );
	UTIL_THROW_IF2(!inserted.second,"Source label set file " << sourceLabelSetFile << " should contain each syntactic label only once.");
	}

	inFile.Close();

	m_sourceLabelsFlag = true;
	}


	void PropertiesConsolidator::ActivatePartsOfSpeechProcessing(const std::string &partsOfSpeechFile)
	{
	Moses::InputFileStream inFile(partsOfSpeechFile);

	// read parts-of-speech vocabulary
	m_partsOfSpeechVocabulary.clear();
	std::string line;
	while (getline(inFile, line)) {
	std::istringstream tokenizer(line);
	std::string label;
	size_t index;
	try {
	tokenizer >> label >> index;
	} catch (const std::exception &e) {
	UTIL_THROW2("Error reading part-of-speech vocabulary file " << partsOfSpeechFile << " .");
	}
	std::pair< std::map<std::string,size_t>::iterator, bool > inserted = m_partsOfSpeechVocabulary.insert( std::pair<std::string,size_t>(label,index) );
	UTIL_THROW_IF2(!inserted.second,"Part-of-speech vocabulary file " << partsOfSpeechFile << " should contain each POS tag only once.");
	}

	inFile.Close();

	m_partsOfSpeechFlag = true;
	}


	void PropertiesConsolidator::ActivateTargetSyntacticPreferencesProcessing(const std::string &targetSyntacticPreferencesLabelSetFile)
	{
	Moses::InputFileStream inFile(targetSyntacticPreferencesLabelSetFile);

	// read target syntactic preferences label set
	m_targetSyntacticPreferencesLabels.clear();
	std::string line;
	while (getline(inFile, line)) {
	std::istringstream tokenizer(line);
	std::string label;
	size_t index;
	try {
	tokenizer >> label >> index;
	} catch (const std::exception &e) {
	UTIL_THROW2("Error reading target syntactic preferences label set file " << targetSyntacticPreferencesLabelSetFile << " .");
	}
	std::pair< std::map<std::string,size_t>::iterator, bool > inserted = m_targetSyntacticPreferencesLabels.insert( std::pair<std::string,size_t>(label,index) );
	UTIL_THROW_IF2(!inserted.second,"Target syntactic preferences label set file " << targetSyntacticPreferencesLabelSetFile << " should contain each syntactic label only once.");
	}

	inFile.Close();

	m_targetSyntacticPreferencesFlag = true;
	}


	void PropertiesConsolidator::ProcessPropertiesString(const std::string &propertiesString, Moses::OutputFileStream& out) const
	{
	if ( propertiesString.empty() ) {
	return;
	}

	std::vector<std::string> toks;
	Moses::TokenizeMultiCharSeparator(toks, propertiesString, "{{");
	for (size_t i = 1; i < toks.size(); ++i) {
	std::string &tok = toks[i];
	if (tok.empty()) {
	continue;
	}
	size_t endPos = tok.rfind("}");
	tok = tok.substr(0, endPos - 1);
	std::vector<std::string> keyValue = Moses::TokenizeFirstOnly(tok, " ");
	assert(keyValue.size() == 2);

	if ( !keyValue[0].compare("SourceLabels") ) {

	if ( m_sourceLabelsFlag ) {

	// SourceLabels property: replace strings with vocabulary indices
	out << " {{" << keyValue[0];
	ProcessSourceLabelsPropertyValue(keyValue[1], out);
	out << "}}";

	} else { // don't process SourceLabels property
	out << " {{" << keyValue[0] << " " << keyValue[1] << "}}";
	}

	} else if ( !keyValue[0].compare("POS") ) {

	/* DO NOTHING (property is not registered in the decoder at the moment)
	if ( m_partsOfSpeechFlag ) {

	// POS property: replace strings with vocabulary indices
	out << " {{" << keyValue[0];
	ProcessPOSPropertyValue(keyValue[1], out);
	out << "}}";

	} else { // don't process POS property
	out << " {{" << keyValue[0] << " " << keyValue[1] << "}}";
	}
	*/

	} else if ( !keyValue[0].compare("TargetPreferences") ) {

	if ( m_targetSyntacticPreferencesFlag ) {

	// TargetPreferences property: replace strings with vocabulary indices
	out << " {{" << keyValue[0];
	ProcessTargetSyntacticPreferencesPropertyValue(keyValue[1], out);
	out << "}}";

	} else { // don't process TargetPreferences property
	out << " {{" << keyValue[0] << " " << keyValue[1] << "}}";
	}

	} else {

	// output other property
	out << " {{" << keyValue[0] << " " << keyValue[1] << "}}";
	}
	}
	}


	void PropertiesConsolidator::ProcessSourceLabelsPropertyValue(const std::string &value, Moses::OutputFileStream& out) const
	{
	// SourceLabels property: replace strings with vocabulary indices
	std::istringstream tokenizer(value);

	size_t nNTs;
	double totalCount;

	if (! (tokenizer >> nNTs)) { // first token: number of non-terminals (incl. left-hand side)
	UTIL_THROW2("Not able to read number of non-terminals from SourceLabels property. "
	<< "Flawed SourceLabels property?");
	}
	assert( nNTs > 0 );
	out << " " << nNTs;

	if (! (tokenizer >> totalCount)) { // second token: overall rule count
	UTIL_THROW2("Not able to read overall rule count from SourceLabels property. "
	<< "Flawed SourceLabels property?");
	}
	assert( totalCount > 0.0 );
	out << " " << totalCount;

	while (tokenizer.peek() != EOF) {
	try {

	size_t numberOfLHSsGivenRHS = std::numeric_limits<std::size_t>::max();

	std::string token;

	if (nNTs > 1) { // rule has right-hand side non-terminals, i.e. it's a hierarchical rule
	for (size_t i=0; i<nNTs-1; ++i) { // RHS source non-terminal labels
	tokenizer >> token; // RHS source non-terminal label
	std::map<std::string,size_t>::const_iterator found = m_sourceLabels.find(token);
	UTIL_THROW_IF2(found == m_sourceLabels.end(), "Label \"" << token << "\" from the phrase table not found in given label set.");
	out << " " << found->second;
	}

	tokenizer >> token; // sourceLabelsRHSCount
	out << " " << token;

	tokenizer >> numberOfLHSsGivenRHS;
	out << " " << numberOfLHSsGivenRHS;
	}

	for (size_t i=0; i<numberOfLHSsGivenRHS && tokenizer.peek()!=EOF; ++i) { // LHS source non-terminal labels seen with this RHS
	tokenizer >> token; // LHS source non-terminal label
	std::map<std::string,size_t>::const_iterator found = m_sourceLabels.find(token);
	UTIL_THROW_IF2(found == m_sourceLabels.end() ,"Label \"" << token << "\" from the phrase table not found in given label set.");
	out << " " << found->second;

	tokenizer >> token; // ruleSourceLabelledCount
	out << " " << token;
	}

	} catch (const std::exception &e) {
	UTIL_THROW2("Flawed item in SourceLabels property?");
	}
	}
	}


	void PropertiesConsolidator::ProcessPOSPropertyValue(const std::string &value, Moses::OutputFileStream& out) const
	{
	std::istringstream tokenizer(value);
	while (tokenizer.peek() != EOF) {
	std::string token;
	tokenizer >> token;
	std::map<std::string,size_t>::const_iterator found = m_partsOfSpeechVocabulary.find(token);
	UTIL_THROW_IF2(found == m_partsOfSpeechVocabulary.end() ,"Part-of-speech \"" << token << "\" from the phrase table not found in given part-of-speech vocabulary.");
	out << " " << found->second;
	}
	}


	bool PropertiesConsolidator::GetPOSPropertyValueFromPropertiesString(const std::string &propertiesString, std::vector<std::string>& out) const
	{
	out.clear();
	if ( propertiesString.empty() ) {
	return false;
	}

	std::vector<std::string> toks;
	Moses::TokenizeMultiCharSeparator(toks, propertiesString, "{{");
	for (size_t i = 1; i < toks.size(); ++i) {
	std::string &tok = toks[i];
	if (tok.empty()) {
	continue;
	}
	size_t endPos = tok.rfind("}");
	tok = tok.substr(0, endPos - 1);
	std::vector<std::string> keyValue = Moses::TokenizeFirstOnly(tok, " ");
	assert(keyValue.size() == 2);

	if ( !keyValue[0].compare("POS") ) {
	std::istringstream tokenizer(keyValue[1]);
	while (tokenizer.peek() != EOF) {
	std::string token;
	tokenizer >> token;
	out.push_back(token);
	}
	return true;
	}
	}

	return false;
	}


	void PropertiesConsolidator::ProcessTargetSyntacticPreferencesPropertyValue(const std::string &value, Moses::OutputFileStream& out) const
	{
	// TargetPreferences property: replace strings with vocabulary indices
	std::istringstream tokenizer(value);

	size_t nNTs;
	double totalCount;

	if (! (tokenizer >> nNTs)) { // first token: number of non-terminals (incl. left-hand side)
	UTIL_THROW2("Not able to read number of non-terminals from TargetPreferences property. "
	<< "Flawed TargetPreferences property?");
	}
	assert( nNTs > 0 );
	out << " " << nNTs;

	if (! (tokenizer >> totalCount)) { // second token: overall rule count
	UTIL_THROW2("Not able to read overall rule count from TargetPreferences property. "
	<< "Flawed TargetPreferences property?");
	}
	assert( totalCount > 0.0 );
	out << " " << totalCount;

	while (tokenizer.peek() != EOF) {
	try {

	size_t numberOfLHSsGivenRHS = std::numeric_limits<std::size_t>::max();

	std::string token;

	if (nNTs > 1) { // rule has right-hand side non-terminals, i.e. it's a hierarchical rule
	for (size_t i=0; i<nNTs-1; ++i) { // RHS target preference non-terminal labels
	tokenizer >> token; // RHS target preference non-terminal label
	std::map<std::string,size_t>::const_iterator found = m_targetSyntacticPreferencesLabels.find(token);
	UTIL_THROW_IF2(found == m_targetSyntacticPreferencesLabels.end(), "Label \"" << token << "\" from the phrase table not found in given label set.");
	out << " " << found->second;
	}

	tokenizer >> token; // targetPreferenceRHSCount
	out << " " << token;

	tokenizer >> numberOfLHSsGivenRHS;
	out << " " << numberOfLHSsGivenRHS;
	}

	for (size_t i=0; i<numberOfLHSsGivenRHS && tokenizer.peek()!=EOF; ++i) { // LHS target preference non-terminal labels seen with this RHS
	tokenizer >> token; // LHS target preference non-terminal label
	std::map<std::string,size_t>::const_iterator found = m_targetSyntacticPreferencesLabels.find(token);
	UTIL_THROW_IF2(found == m_targetSyntacticPreferencesLabels.end() ,"Label \"" << token << "\" from the phrase table not found in given label set.");
	out << " " << found->second;

	tokenizer >> token; // ruleTargetPreferenceLabelledCount
	out << " " << token;
	}

	} catch (const std::exception &e) {
	UTIL_THROW2("Flawed item in TargetPreferences property?");
	}
	}
	}


	} // namespace MosesTraining