wiki-cased / tools /mosesdecoder /moses /FF /DynamicCacheBasedLanguageModel.cpp

Upload folder using huggingface_hub

41f6dd8 verified almost 2 years ago

15.7 kB

#include <utility>
#include "moses/StaticData.h"
#include "moses/InputFileStream.h"
#include "DynamicCacheBasedLanguageModel.h"

namespace Moses
{

std::map< const std::string, DynamicCacheBasedLanguageModel * > DynamicCacheBasedLanguageModel::s_instance_map;
DynamicCacheBasedLanguageModel *DynamicCacheBasedLanguageModel::s_instance = NULL;

DynamicCacheBasedLanguageModel::DynamicCacheBasedLanguageModel(const std::string &line)
  : StatelessFeatureFunction(1, line)
{
  VERBOSE(2,"Initializing DynamicCacheBasedLanguageModel feature..." << std::endl);

  m_query_type = CBLM_QUERY_TYPE_ALLSUBSTRINGS;
  m_score_type = CBLM_SCORE_TYPE_HYPERBOLA;
  m_maxAge = 1000;
  m_name = "default";
  m_constant = false;

  ReadParameters();
  UTIL_THROW_IF2(s_instance_map.find(m_name) != s_instance_map.end(), "Only 1 DynamicCacheBasedLanguageModel feature named " + m_name + " is allowed");
  s_instance_map[m_name] = this;
  s_instance = this; //for back compatibility

  SetPreComputedScores();
}

DynamicCacheBasedLanguageModel::~DynamicCacheBasedLanguageModel() {};

void DynamicCacheBasedLanguageModel::SetPreComputedScores()
{
#ifdef WITH_THREADS
  boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif
  precomputedScores.clear();
  for (unsigned int i=0; i<m_maxAge; i++) {
    precomputedScores.push_back(decaying_score(i));
  }

  if ( m_score_type == CBLM_SCORE_TYPE_HYPERBOLA
       || m_score_type == CBLM_SCORE_TYPE_POWER
       || m_score_type == CBLM_SCORE_TYPE_EXPONENTIAL
       || m_score_type == CBLM_SCORE_TYPE_COSINE ) {
    precomputedScores.push_back(decaying_score(m_maxAge));
  } else { // m_score_type = CBLM_SCORE_TYPE_XXXXXXXXX_REWARD
    precomputedScores.push_back(0.0);
  }
  m_lower_score = precomputedScores[m_maxAge];
  VERBOSE(3, "SetPreComputedScores(): lower_age:|" << m_maxAge << "| lower_score:|" << m_lower_score << "|" << std::endl);
}

float DynamicCacheBasedLanguageModel::GetPreComputedScores(const unsigned int age)
{
  VERBOSE(2, "float DynamicCacheBasedLanguageModel::GetPreComputedScores" << std::endl);
  VERBOSE(2, "age:|"<< age << "|" << std::endl);

  if (age < m_maxAge) {
    return precomputedScores.at(age);
  } else {
    VERBOSE(2, "is to big reduced to m)_maxAge:|"<< m_maxAge << "|" << std::endl);
    return precomputedScores.at(m_maxAge);
  }
}

void DynamicCacheBasedLanguageModel::SetParameter(const std::string& key, const std::string& value)
{
  VERBOSE(2, "DynamicCacheBasedLanguageModel::SetParameter key:|" << key << "| value:|" << value << "|" << std::endl);
  if (key == "cblm-query-type") {
    SetQueryType(Scan<size_t>(value));
  } else if (key == "cblm-score-type") {
    SetScoreType(Scan<size_t>(value));
  } else if (key == "cblm-max-age") {
    SetMaxAge(Scan<unsigned int>(value));
  } else if (key == "cblm-file") {
    m_initfiles = Scan<std::string>(value);
  } else if (key == "cblm-name") {
    m_name = Scan<std::string>(value);
  } else if (key == "cblm-constant") {
    m_constant = Scan<bool>(value);
  } else {
    StatelessFeatureFunction::SetParameter(key, value);
  }
}

void DynamicCacheBasedLanguageModel::EvaluateInIsolation(const Phrase &sp
    , const TargetPhrase &tp
    , ScoreComponentCollection &scoreBreakdown
    , ScoreComponentCollection &estimatedScores) const
{
  float score = m_lower_score;
  switch(m_query_type) {
  case CBLM_QUERY_TYPE_WHOLESTRING:
    score = Evaluate_Whole_String(tp);
    break;
  case CBLM_QUERY_TYPE_ALLSUBSTRINGS:
    score = Evaluate_All_Substrings(tp);
    break;
  default:
    UTIL_THROW_IF2(false, "This score type (" << m_query_type << ") is unknown.");
  }

  scoreBreakdown.Assign(this, score);
}

float DynamicCacheBasedLanguageModel::Evaluate_Whole_String(const TargetPhrase& tp) const
{
  //consider all words in the TargetPhrase as one n-gram
  // and compute the decaying_score for the whole n-gram
  // and return this value

  decaying_cache_t::const_iterator it;
  float score = m_lower_score;

  std::string w = "";
  size_t endpos = tp.GetSize();
  for (size_t pos = 0 ; pos < endpos ; ++pos) {
    w += tp.GetWord(pos).GetFactor(0)->GetString().as_string();
    if (pos < endpos - 1) {
      w += " ";
    }
  }
  it = m_cache.find(w);

  VERBOSE(4,"cblm::Evaluate_Whole_String: searching w:|" << w << "|" << std::endl);
  if (it != m_cache.end()) { //found!
    score = ((*it).second).second;
    VERBOSE(4,"cblm::Evaluate_Whole_String: found w:|" << w << "|" << std::endl);
  }

  VERBOSE(4,"cblm::Evaluate_Whole_String: returning score:|" << score << "|" << std::endl);
  return score;
}

float DynamicCacheBasedLanguageModel::Evaluate_All_Substrings(const TargetPhrase& tp) const
{
  //loop over all n-grams in the TargetPhrase (no matter of n)
  //and compute the decaying_score for all words
  //and return their sum

  decaying_cache_t::const_iterator it;
  float score = 0.0;

  for (size_t startpos = 0 ; startpos < tp.GetSize() ; ++startpos) {
    std::string w = "";
    for (size_t endpos = startpos; endpos < tp.GetSize() ; ++endpos) {
      w += tp.GetWord(endpos).GetFactor(0)->GetString().as_string();
      it = m_cache.find(w);

      if (it != m_cache.end()) { //found!
        score += ((*it).second).second;
        VERBOSE(3,"cblm::Evaluate_All_Substrings: found w:|" << w << "| actual score:|" << ((*it).second).second << "| score:|" << score << "|" << std::endl);
      } else {
        score += m_lower_score;
      }

      if (endpos == startpos) {
        w += " ";
      }

    }
  }
  VERBOSE(3,"cblm::Evaluate_All_Substrings: returning score:|" << score << "|" << std::endl);
  return score;
}

void DynamicCacheBasedLanguageModel::Print() const
{
#ifdef WITH_THREADS
  boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
#endif
  decaying_cache_t::const_iterator it;
  std::cout << "Content of the cache of Cache-Based Language Model" << std::endl;
  std::cout << "Size of the cache of Cache-Based Language Model:|" << m_cache.size() << "|" << std::endl;
  for ( it=m_cache.begin() ; it != m_cache.end(); it++ ) {
    std::cout << "word:|" << (*it).first << "| age:|" << ((*it).second).first << "| score:|" << ((*it).second).second << "|" << std::endl;
  }
}

void DynamicCacheBasedLanguageModel::Decay()
{
#ifdef WITH_THREADS
  boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif
  decaying_cache_t::iterator it;

  unsigned int age;
  float score;
  for ( it=m_cache.begin() ; it != m_cache.end(); it++ ) {
    age=((*it).second).first + 1;
    if (age > m_maxAge) {
      m_cache.erase(it);
      it--;
    } else {
      score = GetPreComputedScores(age);
//      score = decaying_score(age);
      decaying_cache_value_t p (age, score);
      (*it).second = p;
    }
  }
}

void DynamicCacheBasedLanguageModel::Update(std::vector<std::string> words, int age)
{
#ifdef WITH_THREADS
  boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif
  VERBOSE(3,"words.size():|" << words.size() << "|" << std::endl);
  for (size_t j=0; j<words.size(); j++) {
    words[j] = Trim(words[j]);
//    VERBOSE(3,"CacheBasedLanguageModel::Update   word[" << j << "]:"<< words[j] << " age:" << age << " decaying_score(age):" << decaying_score(age) << std::endl);
//    decaying_cache_value_t p (age,decaying_score(age));
    VERBOSE(3,"CacheBasedLanguageModel::Update   word[" << j << "]:"<< words[j] << " age:" << age << " GetPreComputedScores(age):" << GetPreComputedScores(age) << std::endl);
    decaying_cache_value_t p (age,GetPreComputedScores(age));
    std::pair<std::string, decaying_cache_value_t> e (words[j],p);
    m_cache.erase(words[j]); //always erase the element (do nothing if the entry does not exist)
    m_cache.insert(e); //insert the entry
  }
}

void DynamicCacheBasedLanguageModel::ClearEntries(std::string &entries)
{
  if (entries != "") {
    VERBOSE(3,"entries:|" << entries << "|" << std::endl);
    std::vector<std::string> elements = TokenizeMultiCharSeparator(entries, "||");
    VERBOSE(3,"elements.size() after:|" << elements.size() << "|" << std::endl);
    ClearEntries(elements);
  }
}

void DynamicCacheBasedLanguageModel::ClearEntries(std::vector<std::string> words)
{
#ifdef WITH_THREADS
  boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif
  VERBOSE(3,"words.size():|" << words.size() << "|" << std::endl);
  for (size_t j=0; j<words.size(); j++) {
    words[j] = Trim(words[j]);
    VERBOSE(3,"CacheBasedLanguageModel::ClearEntries   word[" << j << "]:"<< words[j] << std::endl);
    m_cache.erase(words[j]); //always erase the element (do nothing if the entry does not exist)
  }
}

void DynamicCacheBasedLanguageModel::Insert(std::string &entries)
{
  if (entries != "") {
    VERBOSE(3,"entries:|" << entries << "|" << std::endl);
    std::vector<std::string> elements = TokenizeMultiCharSeparator(entries, "||");
    VERBOSE(3,"elements.size() after:|" << elements.size() << "|" << std::endl);
    Insert(elements);
  }
}

void DynamicCacheBasedLanguageModel::Insert(std::vector<std::string> ngrams)
{
  VERBOSE(3,"DynamicCacheBasedLanguageModel Insert ngrams.size():|" << ngrams.size() << "|" << std::endl);
  if (m_constant == false) {
    Decay();
  }
  Update(ngrams,1);
  IFVERBOSE(3) Print();
}

void DynamicCacheBasedLanguageModel::ExecuteDlt(std::map<std::string, std::string> dlt_meta)
{
  if (dlt_meta.find("cblm") != dlt_meta.end()) {
    Insert(dlt_meta["cblm"]);
  }
  if (dlt_meta.find("cblm-command") != dlt_meta.end()) {
    Execute(dlt_meta["cblm-command"]);
  }
  if (dlt_meta.find("cblm-file") != dlt_meta.end()) {
    Load(dlt_meta["cblm-file"]);
  }
  if (dlt_meta.find("cblm-clear-entries") != dlt_meta.end()) {
    ClearEntries(dlt_meta["cblm-clear-entries"]);
  }
  if (dlt_meta.find("cblm-clear-all") != dlt_meta.end()) {
    Clear();
  }

}

void DynamicCacheBasedLanguageModel::Execute(std::string command)
{
  VERBOSE(2,"DynamicCacheBasedLanguageModel::Execute(std::string command:|" << command << "|" << std::endl);
  std::vector<std::string> commands = Tokenize(command, "||");
  Execute(commands);
}

void DynamicCacheBasedLanguageModel::Execute(std::vector<std::string> commands)
{
  for (size_t j=0; j<commands.size(); j++) {
    Execute_Single_Command(commands[j]);
  }
  IFVERBOSE(2) Print();
}

void DynamicCacheBasedLanguageModel::Execute_Single_Command(std::string command)
{
  VERBOSE(2,"CacheBasedLanguageModel::Execute_Single_Command(std::string command:|" << command << "|" << std::endl);
  if (command == "clear") {
    VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "|. Cache cleared." << std::endl);
    Clear();
  } else if (command == "settype_wholestring") {
    VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "|. Query type set to " << CBLM_QUERY_TYPE_WHOLESTRING << " (CBLM_QUERY_TYPE_WHOLESTRING)." << std::endl);
    SetQueryType(CBLM_QUERY_TYPE_WHOLESTRING);
  } else if (command == "settype_allsubstrings") {
    VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "|. Query type set to " << CBLM_QUERY_TYPE_ALLSUBSTRINGS << " (CBLM_QUERY_TYPE_ALLSUBSTRINGS)." << std::endl);
    SetQueryType(CBLM_QUERY_TYPE_ALLSUBSTRINGS);
  } else {
    VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "| is unknown. Skipped." << std::endl);
  }
}

void DynamicCacheBasedLanguageModel::Clear()
{
#ifdef WITH_THREADS
  boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif
  m_cache.clear();
}

void DynamicCacheBasedLanguageModel::Load(AllOptions::ptr const& opts)
{
  m_options = opts;
//  SetPreComputedScores();
  VERBOSE(2,"DynamicCacheBasedLanguageModel::Load()" << std::endl);
  Load(m_initfiles);
}

void DynamicCacheBasedLanguageModel::Load(const std::string filestr)
{
  VERBOSE(2,"DynamicCacheBasedLanguageModel::Load(const std::string filestr)" << std::endl);
//  std::vector<std::string> files = Tokenize(m_initfiles, "||");
  std::vector<std::string> files = Tokenize(filestr, "||");
  Load_Multiple_Files(files);
}


void DynamicCacheBasedLanguageModel::Load_Multiple_Files(std::vector<std::string> files)
{
  VERBOSE(2,"DynamicCacheBasedLanguageModel::Load_Multiple_Files(std::vector<std::string> files)" << std::endl);
  for(size_t j = 0; j < files.size(); ++j) {
    Load_Single_File(files[j]);
  }
}

void DynamicCacheBasedLanguageModel::Load_Single_File(const std::string file)
{
  VERBOSE(2,"DynamicCacheBasedLanguageModel::Load_Single_File(const std::string file)" << std::endl);
  //file format
  //age || n-gram
  //age || n-gram || n-gram || n-gram || ...
  //....
  //each n-gram is a sequence of n words (no matter of n)
  //
  //there is no limit on the size of n
  //
  //entries can be repeated, but the last entry overwrites the previous


  VERBOSE(2,"Loading data from the cache file " << file << std::endl);
  InputFileStream cacheFile(file);

  std::string line;
  int age;
  std::vector<std::string> words;

  while (getline(cacheFile, line)) {
    std::vector<std::string> vecStr = TokenizeMultiCharSeparator( line , "||" );
    if (vecStr.size() >= 2) {
      age = Scan<int>(vecStr[0]);
      vecStr.erase(vecStr.begin());
      Update(vecStr,age);
    } else {
      UTIL_THROW_IF2(false, "The format of the loaded file is wrong: " << line);
    }
  }
  IFVERBOSE(2) Print();
}

void DynamicCacheBasedLanguageModel::SetQueryType(size_t type)
{
#ifdef WITH_THREADS
  boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
#endif

  m_query_type = type;
  if ( m_query_type != CBLM_QUERY_TYPE_WHOLESTRING
       && m_query_type != CBLM_QUERY_TYPE_ALLSUBSTRINGS ) {
    VERBOSE(2, "This query type " << m_query_type << " is unknown. Instead used " << CBLM_QUERY_TYPE_ALLSUBSTRINGS << "." << std::endl);
    m_query_type = CBLM_QUERY_TYPE_ALLSUBSTRINGS;
  }
  VERBOSE(2, "CacheBasedLanguageModel QueryType:  " << m_query_type << std::endl);

};

void DynamicCacheBasedLanguageModel::SetScoreType(size_t type)
{
#ifdef WITH_THREADS
  boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
#endif
  m_score_type = type;
  if ( m_score_type != CBLM_SCORE_TYPE_HYPERBOLA
       && m_score_type != CBLM_SCORE_TYPE_POWER
       && m_score_type != CBLM_SCORE_TYPE_EXPONENTIAL
       && m_score_type != CBLM_SCORE_TYPE_COSINE
       && m_score_type != CBLM_SCORE_TYPE_HYPERBOLA_REWARD
       && m_score_type != CBLM_SCORE_TYPE_POWER_REWARD
       && m_score_type != CBLM_SCORE_TYPE_EXPONENTIAL_REWARD ) {
    VERBOSE(2, "This score type " << m_score_type << " is unknown. Instead used " << CBLM_SCORE_TYPE_HYPERBOLA << "." << std::endl);
    m_score_type = CBLM_SCORE_TYPE_HYPERBOLA;
  }
  VERBOSE(2, "CacheBasedLanguageModel ScoreType:  " << m_score_type << std::endl);
};

void DynamicCacheBasedLanguageModel::SetMaxAge(unsigned int age)
{
#ifdef WITH_THREADS
  boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
#endif
  m_maxAge = age;
  VERBOSE(2, "CacheBasedLanguageModel MaxAge:  " << m_maxAge << std::endl);
};

float DynamicCacheBasedLanguageModel::decaying_score(const unsigned int age)
{
  float sc;
  switch(m_score_type) {
  case CBLM_SCORE_TYPE_HYPERBOLA:
    sc = (float) 1.0/age - 1.0;
    break;
  case CBLM_SCORE_TYPE_POWER:
    sc = (float) pow(age, -0.25) - 1.0;
    break;
  case CBLM_SCORE_TYPE_EXPONENTIAL:
    sc = (age == 1) ? 0.0 : (float) exp( 1.0/age ) / exp(1.0) - 1.0;
    break;
  case CBLM_SCORE_TYPE_COSINE:
    sc = (float) cos( (age-1) * (PI/2) / m_maxAge ) - 1.0;
    break;
  case CBLM_SCORE_TYPE_HYPERBOLA_REWARD:
    sc = (float) 1.0/age;
    break;
  case CBLM_SCORE_TYPE_POWER_REWARD:
    sc = (float) pow(age, -0.25);
    break;
  case CBLM_SCORE_TYPE_EXPONENTIAL_REWARD:
    sc = (age == 1) ? 1.0 : (float) exp( 1.0/age ) / exp(1.0);
    break;
  default:
    sc = -1.0;
  }
  return sc;
}
}

	#include <utility>
	#include "moses/StaticData.h"
	#include "moses/InputFileStream.h"
	#include "DynamicCacheBasedLanguageModel.h"

	namespace Moses
	{

	std::map< const std::string, DynamicCacheBasedLanguageModel * > DynamicCacheBasedLanguageModel::s_instance_map;
	DynamicCacheBasedLanguageModel *DynamicCacheBasedLanguageModel::s_instance = NULL;

	DynamicCacheBasedLanguageModel::DynamicCacheBasedLanguageModel(const std::string &line)
	: StatelessFeatureFunction(1, line)
	{
	VERBOSE(2,"Initializing DynamicCacheBasedLanguageModel feature..." << std::endl);

	m_query_type = CBLM_QUERY_TYPE_ALLSUBSTRINGS;
	m_score_type = CBLM_SCORE_TYPE_HYPERBOLA;
	m_maxAge = 1000;
	m_name = "default";
	m_constant = false;

	ReadParameters();
	UTIL_THROW_IF2(s_instance_map.find(m_name) != s_instance_map.end(), "Only 1 DynamicCacheBasedLanguageModel feature named " + m_name + " is allowed");
	s_instance_map[m_name] = this;
	s_instance = this; //for back compatibility

	SetPreComputedScores();
	}

	DynamicCacheBasedLanguageModel::~DynamicCacheBasedLanguageModel() {};

	void DynamicCacheBasedLanguageModel::SetPreComputedScores()
	{
	#ifdef WITH_THREADS
	boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
	#endif
	precomputedScores.clear();
	for (unsigned int i=0; i<m_maxAge; i++) {
	precomputedScores.push_back(decaying_score(i));
	}

	if ( m_score_type == CBLM_SCORE_TYPE_HYPERBOLA
	\|\| m_score_type == CBLM_SCORE_TYPE_POWER
	\|\| m_score_type == CBLM_SCORE_TYPE_EXPONENTIAL
	\|\| m_score_type == CBLM_SCORE_TYPE_COSINE ) {
	precomputedScores.push_back(decaying_score(m_maxAge));
	} else { // m_score_type = CBLM_SCORE_TYPE_XXXXXXXXX_REWARD
	precomputedScores.push_back(0.0);
	}
	m_lower_score = precomputedScores[m_maxAge];
	VERBOSE(3, "SetPreComputedScores(): lower_age:\|" << m_maxAge << "\| lower_score:\|" << m_lower_score << "\|" << std::endl);
	}

	float DynamicCacheBasedLanguageModel::GetPreComputedScores(const unsigned int age)
	{
	VERBOSE(2, "float DynamicCacheBasedLanguageModel::GetPreComputedScores" << std::endl);
	VERBOSE(2, "age:\|"<< age << "\|" << std::endl);

	if (age < m_maxAge) {
	return precomputedScores.at(age);
	} else {
	VERBOSE(2, "is to big reduced to m)_maxAge:\|"<< m_maxAge << "\|" << std::endl);
	return precomputedScores.at(m_maxAge);
	}
	}

	void DynamicCacheBasedLanguageModel::SetParameter(const std::string& key, const std::string& value)
	{
	VERBOSE(2, "DynamicCacheBasedLanguageModel::SetParameter key:\|" << key << "\| value:\|" << value << "\|" << std::endl);
	if (key == "cblm-query-type") {
	SetQueryType(Scan<size_t>(value));
	} else if (key == "cblm-score-type") {
	SetScoreType(Scan<size_t>(value));
	} else if (key == "cblm-max-age") {
	SetMaxAge(Scan<unsigned int>(value));
	} else if (key == "cblm-file") {
	m_initfiles = Scan<std::string>(value);
	} else if (key == "cblm-name") {
	m_name = Scan<std::string>(value);
	} else if (key == "cblm-constant") {
	m_constant = Scan<bool>(value);
	} else {
	StatelessFeatureFunction::SetParameter(key, value);
	}
	}

	void DynamicCacheBasedLanguageModel::EvaluateInIsolation(const Phrase &sp
	, const TargetPhrase &tp
	, ScoreComponentCollection &scoreBreakdown
	, ScoreComponentCollection &estimatedScores) const
	{
	float score = m_lower_score;
	switch(m_query_type) {
	case CBLM_QUERY_TYPE_WHOLESTRING:
	score = Evaluate_Whole_String(tp);
	break;
	case CBLM_QUERY_TYPE_ALLSUBSTRINGS:
	score = Evaluate_All_Substrings(tp);
	break;
	default:
	UTIL_THROW_IF2(false, "This score type (" << m_query_type << ") is unknown.");
	}

	scoreBreakdown.Assign(this, score);
	}

	float DynamicCacheBasedLanguageModel::Evaluate_Whole_String(const TargetPhrase& tp) const
	{
	//consider all words in the TargetPhrase as one n-gram
	// and compute the decaying_score for the whole n-gram
	// and return this value

	decaying_cache_t::const_iterator it;
	float score = m_lower_score;

	std::string w = "";
	size_t endpos = tp.GetSize();
	for (size_t pos = 0 ; pos < endpos ; ++pos) {
	w += tp.GetWord(pos).GetFactor(0)->GetString().as_string();
	if (pos < endpos - 1) {
	w += " ";
	}
	}
	it = m_cache.find(w);

	VERBOSE(4,"cblm::Evaluate_Whole_String: searching w:\|" << w << "\|" << std::endl);
	if (it != m_cache.end()) { //found!
	score = ((*it).second).second;
	VERBOSE(4,"cblm::Evaluate_Whole_String: found w:\|" << w << "\|" << std::endl);
	}

	VERBOSE(4,"cblm::Evaluate_Whole_String: returning score:\|" << score << "\|" << std::endl);
	return score;
	}

	float DynamicCacheBasedLanguageModel::Evaluate_All_Substrings(const TargetPhrase& tp) const
	{
	//loop over all n-grams in the TargetPhrase (no matter of n)
	//and compute the decaying_score for all words
	//and return their sum

	decaying_cache_t::const_iterator it;
	float score = 0.0;

	for (size_t startpos = 0 ; startpos < tp.GetSize() ; ++startpos) {
	std::string w = "";
	for (size_t endpos = startpos; endpos < tp.GetSize() ; ++endpos) {
	w += tp.GetWord(endpos).GetFactor(0)->GetString().as_string();
	it = m_cache.find(w);

	if (it != m_cache.end()) { //found!
	score += ((*it).second).second;
	VERBOSE(3,"cblm::Evaluate_All_Substrings: found w:\|" << w << "\| actual score:\|" << ((*it).second).second << "\| score:\|" << score << "\|" << std::endl);
	} else {
	score += m_lower_score;
	}

	if (endpos == startpos) {
	w += " ";
	}

	}
	}
	VERBOSE(3,"cblm::Evaluate_All_Substrings: returning score:\|" << score << "\|" << std::endl);
	return score;
	}

	void DynamicCacheBasedLanguageModel::Print() const
	{
	#ifdef WITH_THREADS
	boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
	#endif
	decaying_cache_t::const_iterator it;
	std::cout << "Content of the cache of Cache-Based Language Model" << std::endl;
	std::cout << "Size of the cache of Cache-Based Language Model:\|" << m_cache.size() << "\|" << std::endl;
	for ( it=m_cache.begin() ; it != m_cache.end(); it++ ) {
	std::cout << "word:\|" << (it).first << "\| age:\|" << ((it).second).first << "\| score:\|" << ((*it).second).second << "\|" << std::endl;
	}
	}

	void DynamicCacheBasedLanguageModel::Decay()
	{
	#ifdef WITH_THREADS
	boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
	#endif
	decaying_cache_t::iterator it;

	unsigned int age;
	float score;
	for ( it=m_cache.begin() ; it != m_cache.end(); it++ ) {
	age=((*it).second).first + 1;
	if (age > m_maxAge) {
	m_cache.erase(it);
	it--;
	} else {
	score = GetPreComputedScores(age);
	// score = decaying_score(age);
	decaying_cache_value_t p (age, score);
	(*it).second = p;
	}
	}
	}

	void DynamicCacheBasedLanguageModel::Update(std::vector<std::string> words, int age)
	{
	#ifdef WITH_THREADS
	boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
	#endif
	VERBOSE(3,"words.size():\|" << words.size() << "\|" << std::endl);
	for (size_t j=0; j<words.size(); j++) {
	words[j] = Trim(words[j]);
	// VERBOSE(3,"CacheBasedLanguageModel::Update word[" << j << "]:"<< words[j] << " age:" << age << " decaying_score(age):" << decaying_score(age) << std::endl);
	// decaying_cache_value_t p (age,decaying_score(age));
	VERBOSE(3,"CacheBasedLanguageModel::Update word[" << j << "]:"<< words[j] << " age:" << age << " GetPreComputedScores(age):" << GetPreComputedScores(age) << std::endl);
	decaying_cache_value_t p (age,GetPreComputedScores(age));
	std::pair<std::string, decaying_cache_value_t> e (words[j],p);
	m_cache.erase(words[j]); //always erase the element (do nothing if the entry does not exist)
	m_cache.insert(e); //insert the entry
	}
	}

	void DynamicCacheBasedLanguageModel::ClearEntries(std::string &entries)
	{
	if (entries != "") {
	VERBOSE(3,"entries:\|" << entries << "\|" << std::endl);
	std::vector<std::string> elements = TokenizeMultiCharSeparator(entries, "\|\|");
	VERBOSE(3,"elements.size() after:\|" << elements.size() << "\|" << std::endl);
	ClearEntries(elements);
	}
	}

	void DynamicCacheBasedLanguageModel::ClearEntries(std::vector<std::string> words)
	{
	#ifdef WITH_THREADS
	boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
	#endif
	VERBOSE(3,"words.size():\|" << words.size() << "\|" << std::endl);
	for (size_t j=0; j<words.size(); j++) {
	words[j] = Trim(words[j]);
	VERBOSE(3,"CacheBasedLanguageModel::ClearEntries word[" << j << "]:"<< words[j] << std::endl);
	m_cache.erase(words[j]); //always erase the element (do nothing if the entry does not exist)
	}
	}

	void DynamicCacheBasedLanguageModel::Insert(std::string &entries)
	{
	if (entries != "") {
	VERBOSE(3,"entries:\|" << entries << "\|" << std::endl);
	std::vector<std::string> elements = TokenizeMultiCharSeparator(entries, "\|\|");
	VERBOSE(3,"elements.size() after:\|" << elements.size() << "\|" << std::endl);
	Insert(elements);
	}
	}

	void DynamicCacheBasedLanguageModel::Insert(std::vector<std::string> ngrams)
	{
	VERBOSE(3,"DynamicCacheBasedLanguageModel Insert ngrams.size():\|" << ngrams.size() << "\|" << std::endl);
	if (m_constant == false) {
	Decay();
	}
	Update(ngrams,1);
	IFVERBOSE(3) Print();
	}

	void DynamicCacheBasedLanguageModel::ExecuteDlt(std::map<std::string, std::string> dlt_meta)
	{
	if (dlt_meta.find("cblm") != dlt_meta.end()) {
	Insert(dlt_meta["cblm"]);
	}
	if (dlt_meta.find("cblm-command") != dlt_meta.end()) {
	Execute(dlt_meta["cblm-command"]);
	}
	if (dlt_meta.find("cblm-file") != dlt_meta.end()) {
	Load(dlt_meta["cblm-file"]);
	}
	if (dlt_meta.find("cblm-clear-entries") != dlt_meta.end()) {
	ClearEntries(dlt_meta["cblm-clear-entries"]);
	}
	if (dlt_meta.find("cblm-clear-all") != dlt_meta.end()) {
	Clear();
	}

	}

	void DynamicCacheBasedLanguageModel::Execute(std::string command)
	{
	VERBOSE(2,"DynamicCacheBasedLanguageModel::Execute(std::string command:\|" << command << "\|" << std::endl);
	std::vector<std::string> commands = Tokenize(command, "\|\|");
	Execute(commands);
	}

	void DynamicCacheBasedLanguageModel::Execute(std::vector<std::string> commands)
	{
	for (size_t j=0; j<commands.size(); j++) {
	Execute_Single_Command(commands[j]);
	}
	IFVERBOSE(2) Print();
	}

	void DynamicCacheBasedLanguageModel::Execute_Single_Command(std::string command)
	{
	VERBOSE(2,"CacheBasedLanguageModel::Execute_Single_Command(std::string command:\|" << command << "\|" << std::endl);
	if (command == "clear") {
	VERBOSE(2,"CacheBasedLanguageModel Execute command:\|"<< command << "\|. Cache cleared." << std::endl);
	Clear();
	} else if (command == "settype_wholestring") {
	VERBOSE(2,"CacheBasedLanguageModel Execute command:\|"<< command << "\|. Query type set to " << CBLM_QUERY_TYPE_WHOLESTRING << " (CBLM_QUERY_TYPE_WHOLESTRING)." << std::endl);
	SetQueryType(CBLM_QUERY_TYPE_WHOLESTRING);
	} else if (command == "settype_allsubstrings") {
	VERBOSE(2,"CacheBasedLanguageModel Execute command:\|"<< command << "\|. Query type set to " << CBLM_QUERY_TYPE_ALLSUBSTRINGS << " (CBLM_QUERY_TYPE_ALLSUBSTRINGS)." << std::endl);
	SetQueryType(CBLM_QUERY_TYPE_ALLSUBSTRINGS);
	} else {
	VERBOSE(2,"CacheBasedLanguageModel Execute command:\|"<< command << "\| is unknown. Skipped." << std::endl);
	}
	}

	void DynamicCacheBasedLanguageModel::Clear()
	{
	#ifdef WITH_THREADS
	boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
	#endif
	m_cache.clear();
	}

	void DynamicCacheBasedLanguageModel::Load(AllOptions::ptr const& opts)
	{
	m_options = opts;
	// SetPreComputedScores();
	VERBOSE(2,"DynamicCacheBasedLanguageModel::Load()" << std::endl);
	Load(m_initfiles);
	}

	void DynamicCacheBasedLanguageModel::Load(const std::string filestr)
	{
	VERBOSE(2,"DynamicCacheBasedLanguageModel::Load(const std::string filestr)" << std::endl);
	// std::vector<std::string> files = Tokenize(m_initfiles, "\|\|");
	std::vector<std::string> files = Tokenize(filestr, "\|\|");
	Load_Multiple_Files(files);
	}


	void DynamicCacheBasedLanguageModel::Load_Multiple_Files(std::vector<std::string> files)
	{
	VERBOSE(2,"DynamicCacheBasedLanguageModel::Load_Multiple_Files(std::vector<std::string> files)" << std::endl);
	for(size_t j = 0; j < files.size(); ++j) {
	Load_Single_File(files[j]);
	}
	}

	void DynamicCacheBasedLanguageModel::Load_Single_File(const std::string file)
	{
	VERBOSE(2,"DynamicCacheBasedLanguageModel::Load_Single_File(const std::string file)" << std::endl);
	//file format
	//age \|\| n-gram
	//age \|\| n-gram \|\| n-gram \|\| n-gram \|\| ...
	//....
	//each n-gram is a sequence of n words (no matter of n)
	//
	//there is no limit on the size of n
	//
	//entries can be repeated, but the last entry overwrites the previous


	VERBOSE(2,"Loading data from the cache file " << file << std::endl);
	InputFileStream cacheFile(file);

	std::string line;
	int age;
	std::vector<std::string> words;

	while (getline(cacheFile, line)) {
	std::vector<std::string> vecStr = TokenizeMultiCharSeparator( line , "\|\|" );
	if (vecStr.size() >= 2) {
	age = Scan<int>(vecStr[0]);
	vecStr.erase(vecStr.begin());
	Update(vecStr,age);
	} else {
	UTIL_THROW_IF2(false, "The format of the loaded file is wrong: " << line);
	}
	}
	IFVERBOSE(2) Print();
	}

	void DynamicCacheBasedLanguageModel::SetQueryType(size_t type)
	{
	#ifdef WITH_THREADS
	boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
	#endif

	m_query_type = type;
	if ( m_query_type != CBLM_QUERY_TYPE_WHOLESTRING
	&& m_query_type != CBLM_QUERY_TYPE_ALLSUBSTRINGS ) {
	VERBOSE(2, "This query type " << m_query_type << " is unknown. Instead used " << CBLM_QUERY_TYPE_ALLSUBSTRINGS << "." << std::endl);
	m_query_type = CBLM_QUERY_TYPE_ALLSUBSTRINGS;
	}
	VERBOSE(2, "CacheBasedLanguageModel QueryType: " << m_query_type << std::endl);

	};

	void DynamicCacheBasedLanguageModel::SetScoreType(size_t type)
	{
	#ifdef WITH_THREADS
	boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
	#endif
	m_score_type = type;
	if ( m_score_type != CBLM_SCORE_TYPE_HYPERBOLA
	&& m_score_type != CBLM_SCORE_TYPE_POWER
	&& m_score_type != CBLM_SCORE_TYPE_EXPONENTIAL
	&& m_score_type != CBLM_SCORE_TYPE_COSINE
	&& m_score_type != CBLM_SCORE_TYPE_HYPERBOLA_REWARD
	&& m_score_type != CBLM_SCORE_TYPE_POWER_REWARD
	&& m_score_type != CBLM_SCORE_TYPE_EXPONENTIAL_REWARD ) {
	VERBOSE(2, "This score type " << m_score_type << " is unknown. Instead used " << CBLM_SCORE_TYPE_HYPERBOLA << "." << std::endl);
	m_score_type = CBLM_SCORE_TYPE_HYPERBOLA;
	}
	VERBOSE(2, "CacheBasedLanguageModel ScoreType: " << m_score_type << std::endl);
	};

	void DynamicCacheBasedLanguageModel::SetMaxAge(unsigned int age)
	{
	#ifdef WITH_THREADS
	boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
	#endif
	m_maxAge = age;
	VERBOSE(2, "CacheBasedLanguageModel MaxAge: " << m_maxAge << std::endl);
	};

	float DynamicCacheBasedLanguageModel::decaying_score(const unsigned int age)
	{
	float sc;
	switch(m_score_type) {
	case CBLM_SCORE_TYPE_HYPERBOLA:
	sc = (float) 1.0/age - 1.0;
	break;
	case CBLM_SCORE_TYPE_POWER:
	sc = (float) pow(age, -0.25) - 1.0;
	break;
	case CBLM_SCORE_TYPE_EXPONENTIAL:
	sc = (age == 1) ? 0.0 : (float) exp( 1.0/age ) / exp(1.0) - 1.0;
	break;
	case CBLM_SCORE_TYPE_COSINE:
	sc = (float) cos( (age-1) * (PI/2) / m_maxAge ) - 1.0;
	break;
	case CBLM_SCORE_TYPE_HYPERBOLA_REWARD:
	sc = (float) 1.0/age;
	break;
	case CBLM_SCORE_TYPE_POWER_REWARD:
	sc = (float) pow(age, -0.25);
	break;
	case CBLM_SCORE_TYPE_EXPONENTIAL_REWARD:
	sc = (age == 1) ? 1.0 : (float) exp( 1.0/age ) / exp(1.0);
	break;
	default:
	sc = -1.0;
	}
	return sc;
	}
	}