Hugging Face's logo

Quagmire1
/
wiki-cased

Model card Files
xet
 Community
wiki-cased / tools /mosesdecoder /moses /TranslationModel /CompactPT /ListCoders.h
Quagmire1's picture
Quagmire1
Upload folder using huggingface_hub
41f6dd8 verified
Raw History Blame Contribute Delete
7.74 kB
// $Id$
// vim:tabstop=2
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#ifndef moses_ListCoders_h
#define moses_ListCoders_h
#include <cmath>
#include <cassert>
namespace Moses
{
template <typename T = unsigned int>
class VarIntType
{
private:
template <typename IntType, typename OutIt>
static void EncodeSymbol(IntType input, OutIt output) {
if(input == 0) {
*output = 0;
output++;
return;
}
T msb = 1 << (sizeof(T)*8-1);
IntType mask = ~msb;
IntType shift = (sizeof(T)*8-1);
while(input) {
T res = input & mask;
input >>= shift;
if(input)
res |= msb;
*output = res;
output++;
}
};
template <typename InIt, typename IntType>
static void DecodeSymbol(InIt &it, InIt end, IntType &output) {
T msb = 1 << (sizeof(T)*8-1);
IntType shift = (sizeof(T)*8-1);
output = 0;
size_t i = 0;
while(it != end && *it & msb) {
IntType temp = *it & ~msb;
temp <<= shift*i;
output |= temp;
it++;
i++;
}
assert(it != end);
IntType temp = *it;
temp <<= shift*i;
output |= temp;
it++;
}
public:
template <typename InIt, typename OutIt>
static void Encode(InIt it, InIt end, OutIt outIt) {
while(it != end) {
EncodeSymbol(*it, outIt);
it++;
}
}
template <typename InIt, typename OutIt>
static void Decode(InIt &it, InIt end, OutIt outIt) {
while(it != end) {
size_t output;
DecodeSymbol(it, end, output);
*outIt = output;
outIt++;
}
}
template <typename InIt>
static size_t DecodeAndSum(InIt &it, InIt end, size_t num) {
size_t sum = 0;
size_t curr = 0;
while(it != end && curr < num) {
size_t output;
DecodeSymbol(it, end, output);
sum += output;
curr++;
}
return sum;
}
};
typedef VarIntType<unsigned char> VarByte;
typedef VarByte VarInt8;
typedef VarIntType<unsigned short> VarInt16;
typedef VarIntType<unsigned int> VarInt32;
class Simple9
{
private:
typedef unsigned int uint;
template <typename InIt>
inline static void EncodeSymbol(uint &output, InIt it, InIt end) {
uint length = end - it;
uint type = 0;
uint bitlength = 0;
switch(length) {
case 1:
type = 1;
bitlength = 28;
break;
case 2:
type = 2;
bitlength = 14;
break;
case 3:
type = 3;
bitlength = 9;
break;
case 4:
type = 4;
bitlength = 7;
break;
case 5:
type = 5;
bitlength = 5;
break;
case 7:
type = 6;
bitlength = 4;
break;
case 9:
type = 7;
bitlength = 3;
break;
case 14:
type = 8;
bitlength = 2;
break;
case 28:
type = 9;
bitlength = 1;
break;
}
output = 0;
output |= (type << 28);
uint i = 0;
while(it != end) {
UTIL_THROW_IF2(*it > 268435455, "You are trying to encode " << *it
<< " with Simple9. Cannot encode numbers larger than 268435455 (2^28-1)");
uint l = bitlength * (length-i-1);
output |= *it << l;
it++;
i++;
}
}
template <typename OutIt>
static inline void DecodeSymbol(uint input, OutIt outIt) {
uint type = (input >> 28);
uint bitlen = 0;
uint shift = 0;
uint mask = 0;
switch(type) {
case 1:
bitlen = 28;
shift = 0;
mask = 268435455;
break;
case 2:
bitlen = 14;
shift = 14;
mask = 16383;
break;
case 3:
bitlen = 9;
shift = 18;
mask = 511;
break;
case 4:
bitlen = 7;
shift = 21;
mask = 127;
break;
case 5:
bitlen = 5;
shift = 20;
mask = 31;
break;
case 6:
bitlen = 4;
shift = 24;
mask = 15;
break;
case 7:
bitlen = 3;
shift = 24;
mask = 7;
break;
case 8:
bitlen = 2;
shift = 26;
mask = 3;
break;
case 9:
bitlen = 1;
shift = 27;
mask = 1;
break;
}
while(shift > 0) {
*outIt = (input >> shift) & mask;
shift -= bitlen;
outIt++;
}
*outIt = input & mask;
outIt++;
}
static inline size_t DecodeAndSumSymbol(uint input, size_t num, size_t &curr) {
uint type = (input >> 28);
uint bitlen = 0;
uint shift = 0;
uint mask = 0;
switch(type) {
case 1:
bitlen = 28;
shift = 0;
mask = 268435455;
break;
case 2:
bitlen = 14;
shift = 14;
mask = 16383;
break;
case 3:
bitlen = 9;
shift = 18;
mask = 511;
break;
case 4:
bitlen = 7;
shift = 21;
mask = 127;
break;
case 5:
bitlen = 5;
shift = 20;
mask = 31;
break;
case 6:
bitlen = 4;
shift = 24;
mask = 15;
break;
case 7:
bitlen = 3;
shift = 24;
mask = 7;
break;
case 8:
bitlen = 2;
shift = 26;
mask = 3;
break;
case 9:
bitlen = 1;
shift = 27;
mask = 1;
break;
}
size_t sum = 0;
while(shift > 0) {
sum += (input >> shift) & mask;
shift -= bitlen;
if(++curr == num)
return sum;
}
sum += input & mask;
curr++;
return sum;
}
public:
template <typename InIt, typename OutIt>
static void Encode(InIt it, InIt end, OutIt outIt) {
uint parts[] = { 1, 2, 3, 4, 5, 7, 9, 14, 28 };
uint buffer[28];
for(InIt i = it; i < end; i++) {
uint lastbit = 1;
uint lastpos = 0;
uint lastyes = 0;
uint j = 0;
double log2 = log(2);
while(j < 9 && lastpos < 28 && (i+lastpos) < end) {
if(lastpos >= parts[j])
j++;
buffer[lastpos] = *(i + lastpos);
uint reqbit = ceil(log(buffer[lastpos]+1)/log2);
assert(reqbit <= 28);
uint bit = 28/floor(28/reqbit);
if(lastbit < bit)
lastbit = bit;
if(parts[j] > 28/lastbit)
break;
else if(lastpos == parts[j]-1)
lastyes = lastpos;
lastpos++;
}
i += lastyes;
uint length = lastyes + 1;
uint output;
EncodeSymbol(output, buffer, buffer + length);
*outIt = output;
outIt++;
}
}
template <typename InIt, typename OutIt>
static void Decode(InIt &it, InIt end, OutIt outIt) {
while(it != end) {
DecodeSymbol(*it, outIt);
it++;
}
}
template <typename InIt>
static size_t DecodeAndSum(InIt &it, InIt end, size_t num) {
size_t sum = 0;
size_t curr = 0;
while(it != end && curr < num) {
sum += DecodeAndSumSymbol(*it, num, curr);
it++;
}
assert(curr == num);
return sum;
}
};
}
#endif