Download mosesdecoder/phrase-extract/extract-mixed-syntax/ConsistentPhrases.cpp from sleepyhead111/vllm_fairseq: direct link, hf CLI and curl.
- Browser
- Download file 2.79 kB
-
https://huggingface.co/sleepyhead111/vllm_fairseq/resolve/main/mosesdecoder/phrase-extract/extract-mixed-syntax/ConsistentPhrases.cpp
- Command line
-
hf download hf://sleepyhead111/vllm_fairseq/mosesdecoder/phrase-extract/extract-mixed-syntax/ConsistentPhrases.cpp
-
curl -L -o ConsistentPhrases.cpp https://huggingface.co/sleepyhead111/vllm_fairseq/resolve/main/mosesdecoder/phrase-extract/extract-mixed-syntax/ConsistentPhrases.cpp
2.79 kB
| /* | |
| * ConsistentPhrases.cpp | |
| * | |
| * Created on: 20 Feb 2014 | |
| * Author: hieu | |
| */ | |
| using namespace std; | |
| ConsistentPhrases::ConsistentPhrases() | |
| { | |
| } | |
| ConsistentPhrases::~ConsistentPhrases() | |
| { | |
| for (size_t start = 0; start < m_coll.size(); ++start) { | |
| std::vector<Coll> &allSourceStart = m_coll[start]; | |
| for (size_t size = 0; size < allSourceStart.size(); ++size) { | |
| Coll &coll = allSourceStart[size]; | |
| Moses::RemoveAllInColl(coll); | |
| } | |
| } | |
| } | |
| void ConsistentPhrases::Initialize(size_t size) | |
| { | |
| m_coll.resize(size); | |
| for (size_t sourceStart = 0; sourceStart < size; ++sourceStart) { | |
| std::vector<Coll> &allSourceStart = m_coll[sourceStart]; | |
| allSourceStart.resize(size - sourceStart); | |
| } | |
| } | |
| void ConsistentPhrases::Add(int sourceStart, int sourceEnd, | |
| int targetStart, int targetEnd, | |
| const Parameter ¶ms) | |
| { | |
| Coll &coll = m_coll[sourceStart][sourceEnd - sourceStart]; | |
| ConsistentPhrase *cp = new ConsistentPhrase(sourceStart, sourceEnd, | |
| targetStart, targetEnd, | |
| params); | |
| assert(coll.find(cp) == coll.end()); | |
| coll.insert(cp); | |
| } | |
| const ConsistentPhrases::Coll &ConsistentPhrases::GetColl(int sourceStart, int sourceEnd) const | |
| { | |
| const std::vector<Coll> &allSourceStart = m_coll[sourceStart]; | |
| const Coll &ret = allSourceStart[sourceEnd - sourceStart]; | |
| return ret; | |
| } | |
| ConsistentPhrases::Coll &ConsistentPhrases::GetColl(int sourceStart, int sourceEnd) | |
| { | |
| std::vector<Coll> &allSourceStart = m_coll[sourceStart]; | |
| Coll &ret = allSourceStart[sourceEnd - sourceStart]; | |
| return ret; | |
| } | |
| std::string ConsistentPhrases::Debug() const | |
| { | |
| std::stringstream out; | |
| for (size_t start = 0; start < m_coll.size(); ++start) { | |
| const std::vector<Coll> &allSourceStart = m_coll[start]; | |
| for (size_t size = 0; size < allSourceStart.size(); ++size) { | |
| const Coll &coll = allSourceStart[size]; | |
| Coll::const_iterator iter; | |
| for (iter = coll.begin(); iter != coll.end(); ++iter) { | |
| const ConsistentPhrase &consistentPhrase = **iter; | |
| out << consistentPhrase.Debug() << endl; | |
| } | |
| } | |
| } | |
| return out.str(); | |
| } | |
| void ConsistentPhrases::AddHieroNonTerms(const Parameter ¶ms) | |
| { | |
| // add [X] labels everywhere | |
| for (size_t i = 0; i < m_coll.size(); ++i) { | |
| vector<Coll> &inner = m_coll[i]; | |
| for (size_t j = 0; j < inner.size(); ++j) { | |
| ConsistentPhrases::Coll &coll = inner[j]; | |
| ConsistentPhrases::Coll::iterator iter; | |
| for (iter = coll.begin(); iter != coll.end(); ++iter) { | |
| ConsistentPhrase &cp = **iter; | |
| cp.AddNonTerms(params.hieroNonTerm, params.hieroNonTerm); | |
| } | |
| } | |
| } | |
| } | |