llama-cpp-K2-FAST2 / tests /peg-parser /simple-tokenize.cpp

llama.cpp k2-fast2: MBZUAI model/K2Horizon + FAST2 sliding-window override

c59fd93 verified about 1 month ago

863 Bytes

#include "simple-tokenize.h"

std::vector<std::string> simple_tokenize(const std::string & input) {
    std::vector<std::string> result;
    std::string              current;

    for (size_t i = 0; i < input.size(); i++) {
        switch (input[i]) {
            case ' ':
            case '\n':
            case '\t':
            case '{':
            case '}':
            case ',':
            case '[':
            case '"':
            case ']':
            case '.':
            case '<':
            case '>':
            case '=':
            case '/':
                if (!current.empty()) {
                    result.push_back(current);
                    current.clear();
                }
            default:;
        }
        current += input[i];
    }

    if (!current.empty()) {
        result.push_back(current);
    }

    return result;
}

	#include "simple-tokenize.h"

	std::vector<std::string> simple_tokenize(const std::string & input) {
	std::vector<std::string> result;
	std::string current;

	for (size_t i = 0; i < input.size(); i++) {
	switch (input[i]) {
	case ' ':
	case '\n':
	case '\t':
	case '{':
	case '}':
	case ',':
	case '[':
	case '"':
	case ']':
	case '.':
	case '<':
	case '>':
	case '=':
	case '/':
	if (!current.empty()) {
	result.push_back(current);
	current.clear();
	}
	default:;
	}
	current += input[i];
	}

	if (!current.empty()) {
	result.push_back(current);
	}

	return result;
	}