Upload folder using huggingface_hub

41f6dd8 verified almost 2 years ago

6.79 kB

// $Id$
// vim:tabstop=2
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#ifndef moses_MonotonicVector_h
#define moses_MonotonicVector_h

// MonotonicVector - Represents a monotonic increasing function that maps
// positive integers of any size onto a given number type. Each value has to be
// equal or larger than the previous one. Depending on the stepSize it can save
// up to 90% of memory compared to a std::vector<long>. Time complexity is roughly
// constant, in the worst case, however, stepSize times slower than a normal
// std::vector.

#include <vector>
#include <limits>
#include <algorithm>
#include <cstdio>
#include <cassert>

#include "ThrowingFwrite.h"
#include "ListCoders.h"
#include "MmapAllocator.h"

namespace Moses
{

template<typename PosT = size_t, typename NumT = size_t, PosT stepSize = 32,
         template <typename> class Allocator = std::allocator>
class MonotonicVector
{
private:
  typedef std::vector<NumT, Allocator<NumT> > Anchors;
  typedef std::vector<unsigned int, Allocator<unsigned int> > Diffs;

  Anchors m_anchors;
  Diffs m_diffs;
  std::vector<unsigned int> m_tempDiffs;

  size_t m_size;
  PosT m_last;
  bool m_final;

public:
  typedef PosT value_type;

  MonotonicVector() : m_size(0), m_last(0), m_final(false) {}

  size_t size() const {
    return m_size + m_tempDiffs.size();
  }

  PosT at(size_t i) const {
    PosT s = stepSize;
    PosT j = m_anchors[i / s];
    PosT r = i % s;

    typename Diffs::const_iterator it = m_diffs.begin() + j;

    PosT k = 0;
    k += VarInt32::DecodeAndSum(it, m_diffs.end(), 1);
    if(i < m_size)
      k += Simple9::DecodeAndSum(it, m_diffs.end(), r);
    else if(i < m_size + m_tempDiffs.size())
      for(size_t l = 0; l < r; l++)
        k += m_tempDiffs[l];

    return k;
  }

  PosT operator[](PosT i) const {
    return at(i);
  }

  PosT back() const {
    return at(size()-1);
  }

  void push_back(PosT i) {
    assert(m_final != true);

    if(m_anchors.size() == 0 && m_tempDiffs.size() == 0) {
      m_anchors.push_back(0);
      VarInt32::Encode(&i, &i+1, std::back_inserter(m_diffs));
      m_last = i;
      m_size++;

      return;
    }

    if(m_tempDiffs.size() == stepSize-1) {
      Simple9::Encode(m_tempDiffs.begin(), m_tempDiffs.end(),
                      std::back_inserter(m_diffs));
      m_anchors.push_back(m_diffs.size());
      VarInt32::Encode(&i, &i+1, std::back_inserter(m_diffs));

      m_size += m_tempDiffs.size() + 1;
      m_tempDiffs.clear();
    } else {
      PosT last = m_last;
      PosT diff = i - last;
      m_tempDiffs.push_back(diff);
    }
    m_last = i;
  }

  void commit() {
    assert(m_final != true);
    Simple9::Encode(m_tempDiffs.begin(), m_tempDiffs.end(),
                    std::back_inserter(m_diffs));
    m_size += m_tempDiffs.size();
    m_tempDiffs.clear();
    m_final = true;
  }

  size_t usage() {
    return m_diffs.size() * sizeof(unsigned int)
           + m_anchors.size() * sizeof(NumT);
  }

  size_t load(std::FILE* in, bool map = false) {
    size_t byteSize = 0;

    byteSize += fread(&m_final, sizeof(bool), 1, in) * sizeof(bool);
    byteSize += fread(&m_size, sizeof(size_t), 1, in) * sizeof(size_t);
    byteSize += fread(&m_last, sizeof(PosT), 1, in) * sizeof(PosT);

    byteSize += loadVector(m_diffs, in, map);
    byteSize += loadVector(m_anchors, in, map);

    return byteSize;
  }

  template <typename ValueT>
  size_t loadVector(std::vector<ValueT, std::allocator<ValueT> >& v,
                    std::FILE* in, bool map = false) {
    // Can only be read into memory. Mapping not possible with std:allocator.
    assert(map == false);

    size_t byteSize = 0;

    size_t valSize;
    byteSize += std::fread(&valSize, sizeof(size_t), 1, in) * sizeof(size_t);

    v.resize(valSize, 0);
    byteSize += std::fread(&v[0], sizeof(ValueT), valSize, in) * sizeof(ValueT);

    return byteSize;
  }

  template <typename ValueT>
  size_t loadVector(std::vector<ValueT, MmapAllocator<ValueT> >& v,
                    std::FILE* in, bool map = false) {
    size_t byteSize = 0;

    size_t valSize;
    byteSize += std::fread(&valSize, sizeof(size_t), 1, in) * sizeof(size_t);

    if(map == false) {
      // Read data into temporary file (default constructor of MmapAllocator)
      // and map memory onto temporary file. Can be resized.

      v.resize(valSize, 0);
      byteSize += std::fread(&v[0], sizeof(ValueT), valSize, in) * sizeof(ValueT);
    } else {
      // Map it directly on specified region of file "in" starting at valPos
      // with length valSize * sizeof(ValueT). Mapped region cannot be resized.

      size_t valPos = std::ftell(in);

      Allocator<ValueT> alloc(in, valPos);
      std::vector<ValueT, Allocator<ValueT> > vTemp(alloc);
      vTemp.resize(valSize);
      v.swap(vTemp);

      std::fseek(in, valSize * sizeof(ValueT), SEEK_CUR);
      byteSize += valSize * sizeof(ValueT);
    }

    return byteSize;
  }

  size_t save(std::FILE* out) {
    if(!m_final)
      commit();

    bool byteSize = 0;
    byteSize += ThrowingFwrite(&m_final, sizeof(bool), 1, out) * sizeof(bool);
    byteSize += ThrowingFwrite(&m_size, sizeof(size_t), 1, out) * sizeof(size_t);
    byteSize += ThrowingFwrite(&m_last, sizeof(PosT), 1, out) * sizeof(PosT);

    size_t size = m_diffs.size();
    byteSize += ThrowingFwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t);
    byteSize += ThrowingFwrite(&m_diffs[0], sizeof(unsigned int), size, out) * sizeof(unsigned int);

    size = m_anchors.size();
    byteSize += ThrowingFwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t);
    byteSize += ThrowingFwrite(&m_anchors[0], sizeof(NumT), size, out) * sizeof(NumT);

    return byteSize;
  }

  void swap(MonotonicVector<PosT, NumT, stepSize, Allocator> &mv) {
    if(!m_final)
      commit();

    m_diffs.swap(mv.m_diffs);
    m_anchors.swap(mv.m_anchors);
  }
};

}
#endif

	// $Id$
	// vim:tabstop=2
	/***********************************************************************
	Moses - factored phrase-based language decoder
	Copyright (C) 2006 University of Edinburgh

	This library is free software; you can redistribute it and/or
	modify it under the terms of the GNU Lesser General Public
	License as published by the Free Software Foundation; either
	version 2.1 of the License, or (at your option) any later version.

	This library is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	Lesser General Public License for more details.

	You should have received a copy of the GNU Lesser General Public
	License along with this library; if not, write to the Free Software
	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	***********************************************************************/

	#ifndef moses_MonotonicVector_h
	#define moses_MonotonicVector_h

	// MonotonicVector - Represents a monotonic increasing function that maps
	// positive integers of any size onto a given number type. Each value has to be
	// equal or larger than the previous one. Depending on the stepSize it can save
	// up to 90% of memory compared to a std::vector<long>. Time complexity is roughly
	// constant, in the worst case, however, stepSize times slower than a normal
	// std::vector.

	#include <vector>
	#include <limits>
	#include <algorithm>
	#include <cstdio>
	#include <cassert>

	#include "ThrowingFwrite.h"
	#include "ListCoders.h"
	#include "MmapAllocator.h"

	namespace Moses
	{

	template<typename PosT = size_t, typename NumT = size_t, PosT stepSize = 32,
	template <typename> class Allocator = std::allocator>
	class MonotonicVector
	{
	private:
	typedef std::vector<NumT, Allocator<NumT> > Anchors;
	typedef std::vector<unsigned int, Allocator<unsigned int> > Diffs;

	Anchors m_anchors;
	Diffs m_diffs;
	std::vector<unsigned int> m_tempDiffs;

	size_t m_size;
	PosT m_last;
	bool m_final;

	public:
	typedef PosT value_type;

	MonotonicVector() : m_size(0), m_last(0), m_final(false) {}

	size_t size() const {
	return m_size + m_tempDiffs.size();
	}

	PosT at(size_t i) const {
	PosT s = stepSize;
	PosT j = m_anchors[i / s];
	PosT r = i % s;

	typename Diffs::const_iterator it = m_diffs.begin() + j;

	PosT k = 0;
	k += VarInt32::DecodeAndSum(it, m_diffs.end(), 1);
	if(i < m_size)
	k += Simple9::DecodeAndSum(it, m_diffs.end(), r);
	else if(i < m_size + m_tempDiffs.size())
	for(size_t l = 0; l < r; l++)
	k += m_tempDiffs[l];

	return k;
	}

	PosT operator[](PosT i) const {
	return at(i);
	}

	PosT back() const {
	return at(size()-1);
	}

	void push_back(PosT i) {
	assert(m_final != true);

	if(m_anchors.size() == 0 && m_tempDiffs.size() == 0) {
	m_anchors.push_back(0);
	VarInt32::Encode(&i, &i+1, std::back_inserter(m_diffs));
	m_last = i;
	m_size++;

	return;
	}

	if(m_tempDiffs.size() == stepSize-1) {
	Simple9::Encode(m_tempDiffs.begin(), m_tempDiffs.end(),
	std::back_inserter(m_diffs));
	m_anchors.push_back(m_diffs.size());
	VarInt32::Encode(&i, &i+1, std::back_inserter(m_diffs));

	m_size += m_tempDiffs.size() + 1;
	m_tempDiffs.clear();
	} else {
	PosT last = m_last;
	PosT diff = i - last;
	m_tempDiffs.push_back(diff);
	}
	m_last = i;
	}

	void commit() {
	assert(m_final != true);
	Simple9::Encode(m_tempDiffs.begin(), m_tempDiffs.end(),
	std::back_inserter(m_diffs));
	m_size += m_tempDiffs.size();
	m_tempDiffs.clear();
	m_final = true;
	}

	size_t usage() {
	return m_diffs.size() * sizeof(unsigned int)
	+ m_anchors.size() * sizeof(NumT);
	}

	size_t load(std::FILE* in, bool map = false) {
	size_t byteSize = 0;

	byteSize += fread(&m_final, sizeof(bool), 1, in) * sizeof(bool);
	byteSize += fread(&m_size, sizeof(size_t), 1, in) * sizeof(size_t);
	byteSize += fread(&m_last, sizeof(PosT), 1, in) * sizeof(PosT);

	byteSize += loadVector(m_diffs, in, map);
	byteSize += loadVector(m_anchors, in, map);

	return byteSize;
	}

	template <typename ValueT>
	size_t loadVector(std::vector<ValueT, std::allocator<ValueT> >& v,
	std::FILE* in, bool map = false) {
	// Can only be read into memory. Mapping not possible with std:allocator.
	assert(map == false);

	size_t byteSize = 0;

	size_t valSize;
	byteSize += std::fread(&valSize, sizeof(size_t), 1, in) * sizeof(size_t);

	v.resize(valSize, 0);
	byteSize += std::fread(&v[0], sizeof(ValueT), valSize, in) * sizeof(ValueT);

	return byteSize;
	}

	template <typename ValueT>
	size_t loadVector(std::vector<ValueT, MmapAllocator<ValueT> >& v,
	std::FILE* in, bool map = false) {
	size_t byteSize = 0;

	size_t valSize;
	byteSize += std::fread(&valSize, sizeof(size_t), 1, in) * sizeof(size_t);

	if(map == false) {
	// Read data into temporary file (default constructor of MmapAllocator)
	// and map memory onto temporary file. Can be resized.

	v.resize(valSize, 0);
	byteSize += std::fread(&v[0], sizeof(ValueT), valSize, in) * sizeof(ValueT);
	} else {
	// Map it directly on specified region of file "in" starting at valPos
	// with length valSize * sizeof(ValueT). Mapped region cannot be resized.

	size_t valPos = std::ftell(in);

	Allocator<ValueT> alloc(in, valPos);
	std::vector<ValueT, Allocator<ValueT> > vTemp(alloc);
	vTemp.resize(valSize);
	v.swap(vTemp);

	std::fseek(in, valSize * sizeof(ValueT), SEEK_CUR);
	byteSize += valSize * sizeof(ValueT);
	}

	return byteSize;
	}

	size_t save(std::FILE* out) {
	if(!m_final)
	commit();

	bool byteSize = 0;
	byteSize += ThrowingFwrite(&m_final, sizeof(bool), 1, out) * sizeof(bool);
	byteSize += ThrowingFwrite(&m_size, sizeof(size_t), 1, out) * sizeof(size_t);
	byteSize += ThrowingFwrite(&m_last, sizeof(PosT), 1, out) * sizeof(PosT);

	size_t size = m_diffs.size();
	byteSize += ThrowingFwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t);
	byteSize += ThrowingFwrite(&m_diffs[0], sizeof(unsigned int), size, out) * sizeof(unsigned int);

	size = m_anchors.size();
	byteSize += ThrowingFwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t);
	byteSize += ThrowingFwrite(&m_anchors[0], sizeof(NumT), size, out) * sizeof(NumT);

	return byteSize;
	}

	void swap(MonotonicVector<PosT, NumT, stepSize, Allocator> &mv) {
	if(!m_final)
	commit();

	m_diffs.swap(mv.m_diffs);
	m_anchors.swap(mv.m_anchors);
	}
	};

	}
	#endif