Hugging Face's logo

Buckets:
agent-collaborations
/
hutter-prize-67

Files
xet
agent-collaborations/hutter-prize-67 / decoder-bundle-minimization /work /paq8px-src /src /Models.cpp
soixantesept's picture
soixantesept
download
raw
8.33 kB
#include "Models.hpp"
#include "CharacterNames.hpp"
#include "DummyMixer.hpp"
#include "file/OpenFromMyFolder.hpp"
#include "file/FileDisk.hpp"
/*
relationship between compression level, shared->mem and NormalModel memory use as an example
level shared->mem NormalModel memory use (shared->mem*32)
----- ----------- ----------------------
1 0.125 MB 4 MB
2 0.25 MB 8 MB
3 0.5 MB 16 MB
4 1.0 MB 32 MB
5 2.0 MB 64 MB
6 4.0 MB 128 MB
7 8.0 MB 256 MB
8 16.0 MB 512 MB
9 32.0 MB 1024 MB
10 64.0 MB 2048 MB
11 128.0 MB 4096 MB
12 256.0 MB 8192 MB
*/
Models::Models(Shared* const sh, MixerFactory* mf) : shared(sh), mixerFactory(mf) {}
void Models::trainModelsWhenNeeded() {
//initiate pre-training
if (shared->GetOptionTrainTxt()) {
trainText("english.dic", 3);
trainText("english.exp", 1);
}
if (shared->GetOptionTrainExe()) {
trainExe();
}
}
void Models::trainText(const char* const dictionary, int iterations) {
NormalModel& normalModel = this->normalModel();
WordModel& wordModel = this->wordModel();
DummyMixer mDummy(shared,
NormalModel::MIXERINPUTS + WordModel::MIXERINPUTS_TEXT,
NormalModel::MIXERCONTEXTS_PRE + WordModel::MIXERCONTEXTS,
NormalModel::MIXERCONTEXTSETS_PRE + WordModel::MIXERCONTEXTSETS);
shared->State.blockType = BlockType::TEXT;
INJECT_SHARED_pos
INJECT_SHARED_blockPos
assert(pos == 0 && blockPos == 0);
FileDisk f;
printf("Pre-training models with text...");
OpenFromMyFolder::anotherFile(&f, dictionary);
int c = 0;
int trainingByteCount = 0;
while (iterations-- > 0) {
f.setpos(0);
c = SPACE;
trainingByteCount = 0;
do {
trainingByteCount++;
uint8_t c1 = c == NEW_LINE ? SPACE : c;
if (c != CARRIAGE_RETURN) {
for (int bpos = 0; bpos < 8; bpos++) {
normalModel.mix(mDummy); //update (train) model
wordModel.mix(mDummy); //update (train) model
mDummy.p();
int y = (c1 >> (7 - bpos)) & 1;
shared->update(y, 2048, false);
}
}
// emulate a space before and after each word/expression
// reset models in between
if (c == NEW_LINE) {
normalModel.reset();
wordModel.reset();
for (int bpos = 0; bpos < 8; bpos++) {
normalModel.mix(mDummy); //update (train) model
wordModel.mix(mDummy); //update (train) model
mDummy.p();
int y = (c1 >> (7 - bpos)) & 1;
shared->update(y, 2048, false);
}
}
} while ((c = f.getchar()) != EOF);
}
normalModel.reset();
wordModel.reset();
shared->reset();
printf(" done [%s, %d bytes]\n", dictionary, trainingByteCount);
f.close();
}
void Models::trainExe() {
ExeModel& exeModel = this->exeModel();
DummyMixer mDummy(shared, ExeModel::MIXERINPUTS, ExeModel::MIXERCONTEXTS, ExeModel::MIXERCONTEXTSETS);
INJECT_SHARED_pos
INJECT_SHARED_blockPos
assert(pos == 0 && blockPos == 0);
FileDisk f;
printf("Pre-training x86/x64 model...");
OpenFromMyFolder::myself(&f);
int c = 0;
int trainingByteCount = 0;
do {
trainingByteCount++;
for (uint32_t bpos = 0; bpos < 8; bpos++) {
exeModel.mix(mDummy); //update (train) model
mDummy.p();
int y = (c >> (7 - bpos)) & 1;
shared->update(y, 2048, false);
}
} while ((c = f.getchar()) != EOF);
printf(" done [%d bytes]\n", trainingByteCount);
f.close();
shared->reset();
}
auto Models::normalModel() -> NormalModel & {
static NormalModel instance {shared, shared->mem * 32};
return instance;
}
auto Models::dmcForest() -> DmcForest & {
static DmcForest instance {shared, shared->mem}; /**< Not the actual memory use - see in the model */
return instance;
}
auto Models::similarityModelPair() -> SimilarityModelPair& {
static SimilarityModelPair instance{ shared, shared->mem };
return instance;
}
auto Models::charGroupModel() -> CharGroupModel & {
static CharGroupModel instance {shared, shared->mem / 2};
return instance;
}
auto Models::recordModel() -> RecordModel & {
static RecordModel instance {shared, shared->mem * 2};
return instance;
}
auto Models::sparseBitModel() -> SparseBitModel& {
static SparseBitModel instance{ shared, shared->mem / 4 };
return instance;
}
auto Models::sparseModel() -> SparseModel & {
static SparseModel instance {shared, shared->mem * 4};
return instance;
}
auto Models::matchModel() -> MatchModel & {
static MatchModel instance {shared, shared->mem / 4 /*hashtablesize*/, shared->mem /*mapmemorysize*/ }; /**< Not the actual memory use - see in the model */
return instance;
}
auto Models::sparseMatchModel() -> SparseMatchModel & {
static SparseMatchModel instance {shared, shared->mem};
return instance;
}
auto Models::indirectModel() -> IndirectModel & {
static IndirectModel instance {shared, shared->mem * 2};
return instance;
}
auto Models::chartModel() -> ChartModel& {
static ChartModel instance{ shared, shared->mem * 4 };
return instance;
}
auto Models::textModel() -> TextModel & {
static TextModel instance {shared, shared->mem * 16};
return instance;
}
auto Models::wordModel() -> WordModel & {
static WordModel instance {shared, shared->mem * 16};
return instance;
}
auto Models::nestModel() -> NestModel & {
static NestModel instance {shared, shared->mem};
return instance;
}
auto Models::xmlModel() -> XMLModel & {
static XMLModel instance {shared, shared->mem / 4};
return instance;
}
auto Models::exeModel() -> ExeModel & {
static ExeModel instance {shared, shared->mem * 4};
return instance;
}
auto Models::linearPredictionModel() -> LinearPredictionModel & {
static LinearPredictionModel instance {shared};
return instance;
}
auto Models::jpegModel() -> JpegModel & {
static JpegModel instance {shared, mixerFactory, shared->mem}; /**< Not the actual memory use - see in the model */
return instance;
}
auto Models::image24BitModel() -> Image24BitModel & {
static Image24BitModel instance {shared, shared->mem * 4};
return instance;
}
auto Models::image8BitModel() -> Image8BitModel & {
static Image8BitModel instance {shared, shared->mem * 4};
return instance;
}
auto Models::image4BitModel() -> Image4BitModel & {
static Image4BitModel instance {shared, shared->mem / 2};
return instance;
}
auto Models::image1BitModel() -> Image1BitModel & {
static Image1BitModel instance {shared};
return instance;
}
auto Models::audio8BitModel() -> Audio8BitModel & {
static Audio8BitModel instance {shared};
return instance;
}
auto Models::audio16BitModel() -> Audio16BitModel & {
static Audio16BitModel instance {shared};
return instance;
}
auto Models::decAlphaModel() -> DECAlphaModel & {
static DECAlphaModel instance{ shared };
return instance;
}
//An LSTM model adapts slowly to new contents, so we'll have a separate LSTM model per main content type
auto Models::lstmModelText() -> LstmModelContainer& {
static LstmModelContainer* instance = new LstmModelContainer(shared);
return *instance;
}
auto Models::lstmModelGeneric() -> LstmModelContainer& {
static LstmModelContainer* instance = new LstmModelContainer(shared);
return *instance;
}
auto Models::lstmModelDec() -> LstmModelContainer& {
static LstmModelContainer* instance = new LstmModelContainer(shared);
return *instance;
}
auto Models::lstmModelAudio8() -> LstmModelContainer& {
static LstmModelContainer* instance = new LstmModelContainer(shared);
return *instance;
}
auto Models::lstmModelImage1() -> LstmModelContainer& {
static LstmModelContainer* instance = new LstmModelContainer(shared);
return *instance;
}
auto Models::lstmModelImage4() -> LstmModelContainer& {
static LstmModelContainer* instance = new LstmModelContainer(shared);
return *instance;
}
auto Models::lstmModelImage8() -> LstmModelContainer& {
static LstmModelContainer* instance = new LstmModelContainer(shared);
return *instance;
}
auto Models::lstmModelImage24() -> LstmModelContainer& {
static LstmModelContainer* instance = new LstmModelContainer(shared);
return *instance;
}

Xet Storage Details

Size:
8.33 kB
·
Xet hash:
53ae193348f9349f83b9169f3444d575ff680809e98af66366083a10fda39710

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.