kimi-code / packages /kaos /test /internal.test.ts

Add files using upload-large-folder tool

4e23b01 verified about 1 month ago

10.2 kB

import { PassThrough } from 'node:stream';
import { setImmediate as defer, setTimeout as delay } from 'node:timers/promises';

import { describe, expect, it } from 'vitest';

import { BufferedReadable, decodeTextWithErrors, globPatternToRegex } from '#/internal';

async function collectBytes(readable: AsyncIterable<Uint8Array | string>): Promise<Buffer> {
  const chunks: Buffer[] = [];
  for await (const chunk of readable) {
    chunks.push(Buffer.from(chunk));
  }
  return Buffer.concat(chunks);
}

function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
  return Promise.race([
    promise,
    (async () => {
      await delay(timeoutMs);
      throw new Error(`timed out after ${timeoutMs}ms`);
    })(),
  ]);
}

describe('BufferedReadable', () => {
  it('preserves source backpressure until the consumer starts reading', () => {
    const source = new PassThrough({ highWaterMark: 64 * 1024 });
    const buffered = new BufferedReadable(source);
    const chunk = Buffer.alloc(32 * 1024, 0x61);

    let writes = 0;
    let writeOk = true;
    while (writes < 20 && writeOk) {
      writeOk = source.write(chunk);
      writes++;
    }

    buffered.destroy();
    source.destroy();

    expect(writeOk).toBe(false);
    expect(writes).toBeLessThan(10);
  });

  it('stops pushing when its readable buffer reaches highWaterMark', async () => {
    const source = new PassThrough();
    const chunk = Buffer.alloc(32 * 1024, 0x62);

    for (let index = 0; index < 10; index++) {
      source.write(chunk);
    }
    source.end();

    const buffered = new BufferedReadable(source);
    buffered.read(0);
    await defer();

    expect(buffered.readableLength).toBeLessThanOrEqual(buffered.readableHighWaterMark);

    const output = await collectBytes(buffered);
    expect(output).toHaveLength(chunk.length * 10);
  });

  it('ends async iteration when the source closes without emitting end', async () => {
    const source = new PassThrough();
    const buffered = new BufferedReadable(source);
    const outputPromise = collectBytes(buffered).then((output) => output.toString('utf8'));

    await defer();
    source.write('hello');
    await defer();
    source.destroy();

    await expect(withTimeout(outputPromise, 250)).resolves.toBe('hello');
  });

  it('propagates a source error through destroy()', async () => {
    // When the source emits an 'error' event, BufferedReadable must tear
    // itself down with the same error so consumers see the failure rather
    // than waiting forever for data.
    const source = new PassThrough();
    const buffered = new BufferedReadable(source);
    const boom = new Error('source boom');

    const errorReceived = new Promise<Error>((resolve) => {
      buffered.on('error', (err: Error) => {
        resolve(err);
      });
    });

    source.emit('error', boom);

    const received = await withTimeout(errorReceived, 250);
    expect(received).toBe(boom);
    expect(buffered.destroyed).toBe(true);
  });

  it('destroys the underlying source when the wrapper is destroyed', () => {
    const source = new PassThrough();
    const buffered = new BufferedReadable(source);

    buffered.destroy();

    expect(source.destroyed).toBe(true);
  });
});

describe('decodeTextWithErrors', () => {
  it('decodes utf-16le content under strict mode', () => {
    // Covers the utf16le / ucs2 / ucs-2 alias branches that the readText
    // tests (which only exercise utf-8) never touch.
    const data = Buffer.from('hello', 'utf16le');
    expect(decodeTextWithErrors(data, 'utf16le')).toBe('hello');
  });

  it('accepts the ucs2 and ucs-2 encoding aliases', () => {
    const data = Buffer.from('hello', 'utf16le');
    expect(decodeTextWithErrors(data, 'ucs2')).toBe('hello');
    expect(decodeTextWithErrors(data, 'ucs-2' as BufferEncoding)).toBe('hello');
  });

  it('falls back to Buffer.toString for non-UTF encodings', () => {
    // hex / base64 / latin1 are lossless byte↔character mappings so `errors`
    // has no effect; the helper must take the non-TextDecoder branch.
    const data = Buffer.from([0x68, 0x69]); // 'hi' in latin1
    expect(decodeTextWithErrors(data, 'latin1')).toBe('hi');
    expect(decodeTextWithErrors(data, 'hex')).toBe('6869');
  });

  it('preserves valid U+FFFD when ignoring invalid utf-8 bytes', () => {
    const data = Buffer.concat([
      Buffer.from('A\uFFFDB', 'utf-8'),
      Buffer.from([0xff]),
      Buffer.from('C', 'utf-8'),
    ]);

    expect(decodeTextWithErrors(data, 'utf-8', 'ignore')).toBe('A\uFFFDBC');
  });

  it('matches Python ignore behavior for invalid utf-8 sequence boundaries', () => {
    const cases: Buffer[] = [
      Buffer.from([0xc2, 0x41]),
      Buffer.from([0xe0, 0xa0, 0x41]),
      Buffer.from([0xf0, 0x90, 0x80, 0x41]),
      Buffer.from([0xed, 0xa0, 0x80, 0x41]),
    ];

    for (const data of cases) {
      expect(decodeTextWithErrors(data, 'utf-8', 'ignore')).toBe('A');
    }
  });

  it('preserves valid U+FFFD when ignoring invalid utf-16le code units', () => {
    const data = Buffer.concat([
      Buffer.from('A\uFFFDB', 'utf16le'),
      Buffer.from([0x3d, 0xd8]), // lone high surrogate
      Buffer.from('C', 'utf16le'),
      Buffer.from([0xff]), // trailing half code unit
    ]);

    expect(decodeTextWithErrors(data, 'utf16le', 'ignore')).toBe('A\uFFFDBC');
  });

  it('matches Python ignore behavior for utf-16le surrogate boundaries', () => {
    const emojiPair = Buffer.from([0x3d, 0xd8, 0x00, 0xde]);
    expect(decodeTextWithErrors(emojiPair, 'utf16le', 'ignore')).toBe('\u{1F600}');

    const highThenNormalThenLow = Buffer.from([0x3d, 0xd8, 0x58, 0x00, 0x00, 0xdc]);
    expect(decodeTextWithErrors(highThenNormalThenLow, 'utf16le', 'ignore')).toBe('X');
  });
});

describe('globPatternToRegex additional cases', () => {
  it('converts ? to a single-character class that excludes /', () => {
    const regex = globPatternToRegex('f?o.txt', true);
    expect(regex.test('foo.txt')).toBe(true);
    expect(regex.test('fao.txt')).toBe(true);
    // ? must match exactly one character
    expect(regex.test('fo.txt')).toBe(false);
    // ? must not cross path segments
    expect(regex.test('f/o.txt')).toBe(false);
  });

  it('treats an unclosed [ as a literal bracket', () => {
    // Mirrors Python fnmatch/glob: a bare `[` with no closing `]` is
    // re-emitted as an escaped literal instead of starting a char class.
    const regex = globPatternToRegex('file[', true);
    expect(regex.test('file[')).toBe(true);
    expect(regex.test('file]')).toBe(false);
  });

  it('escapes regex metacharacters in the default branch', () => {
    // `+`, `.`, `(`, `)`, `$`, `^`, `|`, `{`, `}`, `\` must be escaped so
    // `a+b.c$d(e)` in a glob pattern matches that literal filename.
    const regex = globPatternToRegex('a+b.c$d(e)', true);
    expect(regex.test('a+b.c$d(e)')).toBe(true);
    // Without escaping, `+` would make `ab` match via one-or-more repetition.
    expect(regex.test('ab.c$d(e)')).toBe(false);
  });

  it('treats backslash as an escape character for the next literal', () => {
    // `\[` and `\]` should match literal brackets in the filename.
    const regex = globPatternToRegex('special \\[bracket\\].ts', true);
    expect(regex.test('special [bracket].ts')).toBe(true);
    expect(regex.test('special bracket.ts')).toBe(false);
  });

  it('escapes backslashes inside character classes to avoid an unterminated regex', () => {
    // A trailing backslash inside a glob char class (`[abc\\]`) would
    // escape the closing `]` in the resulting regex. We escape it so the
    // regex stays valid and the backslash is treated as a literal member.
    const regex = globPatternToRegex('file[abc\\].txt', true);
    expect(regex.test('filea.txt')).toBe(true);
    expect(regex.test('file\\.txt')).toBe(true);
    expect(regex.test('filez.txt')).toBe(false);
  });
});

describe('globPatternToRegex', () => {
  it('treats [!...] as a negated character class', () => {
    const regex = globPatternToRegex('[!a].txt', true);

    expect(regex.test('a.txt')).toBe(false);
    expect(regex.test('b.txt')).toBe(true);
  });

  it('treats ^ as a literal character inside glob character classes', () => {
    const regex = globPatternToRegex('[^a].txt', true);

    expect(regex.test('^.txt')).toBe(true);
    expect(regex.test('a.txt')).toBe(true);
    expect(regex.test('b.txt')).toBe(false);
  });

  describe('glob semantic compatibility (Python parity)', () => {
    it('treats brace expansion syntax as literal characters', () => {
      const regex = globPatternToRegex('*.{js,ts}', true);

      expect(regex.test('file.js')).toBe(false);
      expect(regex.test('file.ts')).toBe(false);
      expect(regex.test('file.{js,ts}')).toBe(true);
    });

    it('treats leading ^ as a literal character inside character classes', () => {
      const regex = globPatternToRegex('[^a].txt', true);

      expect(regex.test('^.txt')).toBe(true);
      expect(regex.test('a.txt')).toBe(true);
      expect(regex.test('b.txt')).toBe(false);
    });

    it('uses ! for character class negation', () => {
      const regex = globPatternToRegex('[!a].txt', true);

      expect(regex.test('b.txt')).toBe(true);
      expect(regex.test('a.txt')).toBe(false);
    });

    it('matches hidden files with dot-prefixed patterns', () => {
      const regex = globPatternToRegex('.*', true);

      expect(regex.test('.hidden')).toBe(true);
      expect(regex.test('.config')).toBe(true);
    });

    it.skip('Python treats **/foo.txt as recursive; current helper is segment-based and does not implement zero-or-more directories', () => {
      const regex = globPatternToRegex('**/foo.txt', true);

      expect(regex.test('foo.txt')).toBe(true);
      expect(regex.test('a/foo.txt')).toBe(true);
      expect(regex.test('a/b/foo.txt')).toBe(true);
    });

    it('keeps single-star matching to a single path segment', () => {
      const regex = globPatternToRegex('*/foo.txt', true);

      expect(regex.test('a/foo.txt')).toBe(true);
      expect(regex.test('foo.txt')).toBe(false);
      expect(regex.test('a/b/foo.txt')).toBe(false);
    });
  });
});

	import { PassThrough } from 'node:stream';
	import { setImmediate as defer, setTimeout as delay } from 'node:timers/promises';

	import { describe, expect, it } from 'vitest';

	import { BufferedReadable, decodeTextWithErrors, globPatternToRegex } from '#/internal';

	async function collectBytes(readable: AsyncIterable<Uint8Array \| string>): Promise<Buffer> {
	const chunks: Buffer[] = [];
	for await (const chunk of readable) {
	chunks.push(Buffer.from(chunk));
	}
	return Buffer.concat(chunks);
	}

	function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
	return Promise.race([
	promise,
	(async () => {
	await delay(timeoutMs);
	throw new Error(`timed out after ${timeoutMs}ms`);
	})(),
	]);
	}

	describe('BufferedReadable', () => {
	it('preserves source backpressure until the consumer starts reading', () => {
	const source = new PassThrough({ highWaterMark: 64 * 1024 });
	const buffered = new BufferedReadable(source);
	const chunk = Buffer.alloc(32 * 1024, 0x61);

	let writes = 0;
	let writeOk = true;
	while (writes < 20 && writeOk) {
	writeOk = source.write(chunk);
	writes++;
	}

	buffered.destroy();
	source.destroy();

	expect(writeOk).toBe(false);
	expect(writes).toBeLessThan(10);
	});

	it('stops pushing when its readable buffer reaches highWaterMark', async () => {
	const source = new PassThrough();
	const chunk = Buffer.alloc(32 * 1024, 0x62);

	for (let index = 0; index < 10; index++) {
	source.write(chunk);
	}
	source.end();

	const buffered = new BufferedReadable(source);
	buffered.read(0);
	await defer();

	expect(buffered.readableLength).toBeLessThanOrEqual(buffered.readableHighWaterMark);

	const output = await collectBytes(buffered);
	expect(output).toHaveLength(chunk.length * 10);
	});

	it('ends async iteration when the source closes without emitting end', async () => {
	const source = new PassThrough();
	const buffered = new BufferedReadable(source);
	const outputPromise = collectBytes(buffered).then((output) => output.toString('utf8'));

	await defer();
	source.write('hello');
	await defer();
	source.destroy();

	await expect(withTimeout(outputPromise, 250)).resolves.toBe('hello');
	});

	it('propagates a source error through destroy()', async () => {
	// When the source emits an 'error' event, BufferedReadable must tear
	// itself down with the same error so consumers see the failure rather
	// than waiting forever for data.
	const source = new PassThrough();
	const buffered = new BufferedReadable(source);
	const boom = new Error('source boom');

	const errorReceived = new Promise<Error>((resolve) => {
	buffered.on('error', (err: Error) => {
	resolve(err);
	});
	});

	source.emit('error', boom);

	const received = await withTimeout(errorReceived, 250);
	expect(received).toBe(boom);
	expect(buffered.destroyed).toBe(true);
	});

	it('destroys the underlying source when the wrapper is destroyed', () => {
	const source = new PassThrough();
	const buffered = new BufferedReadable(source);

	buffered.destroy();

	expect(source.destroyed).toBe(true);
	});
	});

	describe('decodeTextWithErrors', () => {
	it('decodes utf-16le content under strict mode', () => {
	// Covers the utf16le / ucs2 / ucs-2 alias branches that the readText
	// tests (which only exercise utf-8) never touch.
	const data = Buffer.from('hello', 'utf16le');
	expect(decodeTextWithErrors(data, 'utf16le')).toBe('hello');
	});

	it('accepts the ucs2 and ucs-2 encoding aliases', () => {
	const data = Buffer.from('hello', 'utf16le');
	expect(decodeTextWithErrors(data, 'ucs2')).toBe('hello');
	expect(decodeTextWithErrors(data, 'ucs-2' as BufferEncoding)).toBe('hello');
	});

	it('falls back to Buffer.toString for non-UTF encodings', () => {
	// hex / base64 / latin1 are lossless byte↔character mappings so `errors`
	// has no effect; the helper must take the non-TextDecoder branch.
	const data = Buffer.from([0x68, 0x69]); // 'hi' in latin1
	expect(decodeTextWithErrors(data, 'latin1')).toBe('hi');
	expect(decodeTextWithErrors(data, 'hex')).toBe('6869');
	});

	it('preserves valid U+FFFD when ignoring invalid utf-8 bytes', () => {
	const data = Buffer.concat([
	Buffer.from('A\uFFFDB', 'utf-8'),
	Buffer.from([0xff]),
	Buffer.from('C', 'utf-8'),
	]);

	expect(decodeTextWithErrors(data, 'utf-8', 'ignore')).toBe('A\uFFFDBC');
	});

	it('matches Python ignore behavior for invalid utf-8 sequence boundaries', () => {
	const cases: Buffer[] = [
	Buffer.from([0xc2, 0x41]),
	Buffer.from([0xe0, 0xa0, 0x41]),
	Buffer.from([0xf0, 0x90, 0x80, 0x41]),
	Buffer.from([0xed, 0xa0, 0x80, 0x41]),
	];

	for (const data of cases) {
	expect(decodeTextWithErrors(data, 'utf-8', 'ignore')).toBe('A');
	}
	});

	it('preserves valid U+FFFD when ignoring invalid utf-16le code units', () => {
	const data = Buffer.concat([
	Buffer.from('A\uFFFDB', 'utf16le'),
	Buffer.from([0x3d, 0xd8]), // lone high surrogate
	Buffer.from('C', 'utf16le'),
	Buffer.from([0xff]), // trailing half code unit
	]);

	expect(decodeTextWithErrors(data, 'utf16le', 'ignore')).toBe('A\uFFFDBC');
	});

	it('matches Python ignore behavior for utf-16le surrogate boundaries', () => {
	const emojiPair = Buffer.from([0x3d, 0xd8, 0x00, 0xde]);
	expect(decodeTextWithErrors(emojiPair, 'utf16le', 'ignore')).toBe('\u{1F600}');

	const highThenNormalThenLow = Buffer.from([0x3d, 0xd8, 0x58, 0x00, 0x00, 0xdc]);
	expect(decodeTextWithErrors(highThenNormalThenLow, 'utf16le', 'ignore')).toBe('X');
	});
	});

	describe('globPatternToRegex additional cases', () => {
	it('converts ? to a single-character class that excludes /', () => {
	const regex = globPatternToRegex('f?o.txt', true);
	expect(regex.test('foo.txt')).toBe(true);
	expect(regex.test('fao.txt')).toBe(true);
	// ? must match exactly one character
	expect(regex.test('fo.txt')).toBe(false);
	// ? must not cross path segments
	expect(regex.test('f/o.txt')).toBe(false);
	});

	it('treats an unclosed [ as a literal bracket', () => {
	// Mirrors Python fnmatch/glob: a bare `[` with no closing `]` is
	// re-emitted as an escaped literal instead of starting a char class.
	const regex = globPatternToRegex('file[', true);
	expect(regex.test('file[')).toBe(true);
	expect(regex.test('file]')).toBe(false);
	});

	it('escapes regex metacharacters in the default branch', () => {
	// `+`, `.`, `(`, `)`, `$`, `^`, `\|`, `{`, `}`, `\` must be escaped so
	// `a+b.c$d(e)` in a glob pattern matches that literal filename.
	const regex = globPatternToRegex('a+b.c$d(e)', true);
	expect(regex.test('a+b.c$d(e)')).toBe(true);
	// Without escaping, `+` would make `ab` match via one-or-more repetition.
	expect(regex.test('ab.c$d(e)')).toBe(false);
	});

	it('treats backslash as an escape character for the next literal', () => {
	// `\[` and `\]` should match literal brackets in the filename.
	const regex = globPatternToRegex('special \\[bracket\\].ts', true);
	expect(regex.test('special [bracket].ts')).toBe(true);
	expect(regex.test('special bracket.ts')).toBe(false);
	});

	it('escapes backslashes inside character classes to avoid an unterminated regex', () => {
	// A trailing backslash inside a glob char class (`[abc\\]`) would
	// escape the closing `]` in the resulting regex. We escape it so the
	// regex stays valid and the backslash is treated as a literal member.
	const regex = globPatternToRegex('file[abc\\].txt', true);
	expect(regex.test('filea.txt')).toBe(true);
	expect(regex.test('file\\.txt')).toBe(true);
	expect(regex.test('filez.txt')).toBe(false);
	});
	});

	describe('globPatternToRegex', () => {
	it('treats [!...] as a negated character class', () => {
	const regex = globPatternToRegex('[!a].txt', true);

	expect(regex.test('a.txt')).toBe(false);
	expect(regex.test('b.txt')).toBe(true);
	});

	it('treats ^ as a literal character inside glob character classes', () => {
	const regex = globPatternToRegex('[^a].txt', true);

	expect(regex.test('^.txt')).toBe(true);
	expect(regex.test('a.txt')).toBe(true);
	expect(regex.test('b.txt')).toBe(false);
	});

	describe('glob semantic compatibility (Python parity)', () => {
	it('treats brace expansion syntax as literal characters', () => {
	const regex = globPatternToRegex('*.{js,ts}', true);

	expect(regex.test('file.js')).toBe(false);
	expect(regex.test('file.ts')).toBe(false);
	expect(regex.test('file.{js,ts}')).toBe(true);
	});

	it('treats leading ^ as a literal character inside character classes', () => {
	const regex = globPatternToRegex('[^a].txt', true);

	expect(regex.test('^.txt')).toBe(true);
	expect(regex.test('a.txt')).toBe(true);
	expect(regex.test('b.txt')).toBe(false);
	});

	it('uses ! for character class negation', () => {
	const regex = globPatternToRegex('[!a].txt', true);

	expect(regex.test('b.txt')).toBe(true);
	expect(regex.test('a.txt')).toBe(false);
	});

	it('matches hidden files with dot-prefixed patterns', () => {
	const regex = globPatternToRegex('.*', true);

	expect(regex.test('.hidden')).toBe(true);
	expect(regex.test('.config')).toBe(true);
	});

	it.skip('Python treats **/foo.txt as recursive; current helper is segment-based and does not implement zero-or-more directories', () => {
	const regex = globPatternToRegex('**/foo.txt', true);

	expect(regex.test('foo.txt')).toBe(true);
	expect(regex.test('a/foo.txt')).toBe(true);
	expect(regex.test('a/b/foo.txt')).toBe(true);
	});

	it('keeps single-star matching to a single path segment', () => {
	const regex = globPatternToRegex('*/foo.txt', true);

	expect(regex.test('a/foo.txt')).toBe(true);
	expect(regex.test('foo.txt')).toBe(false);
	expect(regex.test('a/b/foo.txt')).toBe(false);
	});
	});
	});