From c03b2c4501689ead25510dea7d9e39bd258df557 Mon Sep 17 00:00:00 2001 From: John Doe Date: Thu, 8 Jan 2026 20:11:24 +0100 Subject: [PATCH 1/5] refactor: add general file sink logic --- .../utils/src/lib/file-sink-json.int.test.ts | 138 ++++++++ packages/utils/src/lib/file-sink-json.ts | 60 ++++ .../utils/src/lib/file-sink-json.unit.test.ts | 216 +++++++++++++ .../utils/src/lib/file-sink-text.int.test.ts | 184 +++++++++++ packages/utils/src/lib/file-sink-text.ts | 147 +++++++++ .../utils/src/lib/file-sink-text.unit.test.ts | 295 ++++++++++++++++++ packages/utils/src/lib/sink-source.types.ts | 48 +++ 7 files changed, 1088 insertions(+) create mode 100644 packages/utils/src/lib/file-sink-json.int.test.ts create mode 100644 packages/utils/src/lib/file-sink-json.ts create mode 100644 packages/utils/src/lib/file-sink-json.unit.test.ts create mode 100644 packages/utils/src/lib/file-sink-text.int.test.ts create mode 100644 packages/utils/src/lib/file-sink-text.ts create mode 100644 packages/utils/src/lib/file-sink-text.unit.test.ts create mode 100644 packages/utils/src/lib/sink-source.types.ts diff --git a/packages/utils/src/lib/file-sink-json.int.test.ts b/packages/utils/src/lib/file-sink-json.int.test.ts new file mode 100644 index 000000000..c331d8d0a --- /dev/null +++ b/packages/utils/src/lib/file-sink-json.int.test.ts @@ -0,0 +1,138 @@ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { teardownTestFolder } from '@code-pushup/test-utils'; +import { JsonlFileSink, recoverJsonlFile } from './file-sink-json.js'; + +describe('JsonlFileSink integration', () => { + const baseDir = path.join(os.tmpdir(), 'file-sink-json-int-tests'); + const testFile = path.join(baseDir, 'test-data.jsonl'); + + beforeAll(async () => { + await fs.promises.mkdir(baseDir, { recursive: true }); + }); + + beforeEach(async () => { + try { + await fs.promises.unlink(testFile); + } catch { + // File doesn't exist, which is fine + } + }); + + afterAll(async () => { + await teardownTestFolder(baseDir); + }); + + describe('file operations', () => { + const testData = [ + { id: 1, name: 'Alice', active: true }, + { id: 2, name: 'Bob', active: false }, + { id: 3, name: 'Charlie', active: true }, + ]; + + it('should write and read JSONL files', async () => { + const sink = new JsonlFileSink({ filePath: testFile }); + + // Open and write data + sink.open(); + testData.forEach(item => sink.write(item)); + sink.close(); + + expect(fs.existsSync(testFile)).toBe(true); + const fileContent = fs.readFileSync(testFile, 'utf8'); + const lines = fileContent.trim().split('\n'); + expect(lines).toStrictEqual([ + '{"id":1,"name":"Alice","active":true}', + '{"id":2,"name":"Bob","active":false}', + '{"id":3,"name":"Charlie","active":true}', + ]); + + lines.forEach((line, index) => { + const parsed = JSON.parse(line); + expect(parsed).toStrictEqual(testData[index]); + }); + }); + + it('should recover data from JSONL files', async () => { + const jsonlContent = `${testData.map(item => JSON.stringify(item)).join('\n')}\n`; + fs.writeFileSync(testFile, jsonlContent); + + expect(recoverJsonlFile(testFile)).toStrictEqual({ + records: testData, + errors: [], + partialTail: null, + }); + }); + + it('should handle JSONL files with parse errors', async () => { + const mixedContent = + '{"id":1,"name":"Alice"}\n' + + 'invalid json line\n' + + '{"id":2,"name":"Bob"}\n' + + '{"id":3,"name":"Charlie","incomplete":\n'; + + fs.writeFileSync(testFile, mixedContent); + + expect(recoverJsonlFile(testFile)).toStrictEqual({ + records: [ + { id: 1, name: 'Alice' }, + { id: 2, name: 'Bob' }, + ], + errors: [ + expect.objectContaining({ line: 'invalid json line' }), + expect.objectContaining({ + line: '{"id":3,"name":"Charlie","incomplete":', + }), + ], + partialTail: '{"id":3,"name":"Charlie","incomplete":', + }); + }); + + it('should recover data using JsonlFileSink.recover()', async () => { + const sink = new JsonlFileSink({ filePath: testFile }); + sink.open(); + testData.forEach(item => sink.write(item)); + sink.close(); + + expect(sink.recover()).toStrictEqual({ + records: testData, + errors: [], + partialTail: null, + }); + }); + + describe('edge cases', () => { + it('should handle empty files', async () => { + fs.writeFileSync(testFile, ''); + + expect(recoverJsonlFile(testFile)).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + + it('should handle files with only whitespace', async () => { + fs.writeFileSync(testFile, ' \n \n\t\n'); + + expect(recoverJsonlFile(testFile)).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + + it('should handle non-existent files', async () => { + const nonExistentFile = path.join(baseDir, 'does-not-exist.jsonl'); + + expect(recoverJsonlFile(nonExistentFile)).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + }); + }); +}); diff --git a/packages/utils/src/lib/file-sink-json.ts b/packages/utils/src/lib/file-sink-json.ts new file mode 100644 index 000000000..646cd82b1 --- /dev/null +++ b/packages/utils/src/lib/file-sink-json.ts @@ -0,0 +1,60 @@ +import * as fs from 'node:fs'; +import { + type FileOutput, + FileSink, + type FileSinkOptions, + stringDecode, + stringEncode, + stringRecover, +} from './file-sink-text.js'; +import type { RecoverOptions, RecoverResult } from './sink-source.types.js'; + +export const jsonlEncode = < + T extends Record = Record, +>( + input: T, +): FileOutput => JSON.stringify(input); + +export const jsonlDecode = < + T extends Record = Record, +>( + output: FileOutput, +): T => JSON.parse(stringDecode(output)) as T; + +export function recoverJsonlFile< + T extends Record = Record, +>(filePath: string, opts: RecoverOptions = {}): RecoverResult { + return stringRecover(filePath, jsonlDecode, opts); +} + +export class JsonlFileSink< + T extends Record = Record, +> extends FileSink { + constructor(options: FileSinkOptions) { + const { filePath, ...fileOptions } = options; + super({ + ...fileOptions, + filePath, + recover: () => recoverJsonlFile(filePath), + finalize: () => { + // No additional finalization needed for JSONL files + }, + }); + } + + override encode(input: T): FileOutput { + return stringEncode(jsonlEncode(input)); + } + + override decode(output: FileOutput): T { + return jsonlDecode(stringDecode(output)); + } + + override repack(outputPath?: string): void { + const { records } = this.recover(); + fs.writeFileSync( + outputPath ?? this.getFilePath(), + records.map(this.encode).join(''), + ); + } +} diff --git a/packages/utils/src/lib/file-sink-json.unit.test.ts b/packages/utils/src/lib/file-sink-json.unit.test.ts new file mode 100644 index 000000000..a920c8cbe --- /dev/null +++ b/packages/utils/src/lib/file-sink-json.unit.test.ts @@ -0,0 +1,216 @@ +import { vol } from 'memfs'; +import * as fs from 'node:fs'; +import { beforeEach, describe, expect, it } from 'vitest'; +import { MEMFS_VOLUME } from '@code-pushup/test-utils'; +import { + JsonlFileSink, + jsonlDecode, + jsonlEncode, + recoverJsonlFile, +} from './file-sink-json.js'; + +describe('jsonlEncode', () => { + it('should encode object to JSON string', () => { + const obj = { key: 'value', number: 42 }; + expect(jsonlEncode(obj)).toBe(JSON.stringify(obj)); + }); + + it('should handle nested objects', () => { + const obj = { nested: { deep: 'value' }, array: [1, 2, 3] }; + expect(jsonlEncode(obj)).toBe(JSON.stringify(obj)); + }); + + it('should handle empty object', () => { + expect(jsonlEncode({})).toBe('{}'); + }); +}); + +describe('jsonlDecode', () => { + it('should decode JSON string to object', () => { + const obj = { key: 'value', number: 42 }; + const jsonStr = `${JSON.stringify(obj)}\n`; + expect(jsonlDecode(jsonStr)).toStrictEqual(obj); + }); + + it('should handle nested objects', () => { + const obj = { nested: { deep: 'value' }, array: [1, 2, 3] }; + const jsonStr = `${JSON.stringify(obj)}\n`; + expect(jsonlDecode(jsonStr)).toStrictEqual(obj); + }); + + it('should trim whitespace before parsing', () => { + const obj = { key: 'value' }; + const jsonStr = ` ${JSON.stringify(obj)} \n`; + expect(jsonlDecode(jsonStr)).toStrictEqual(obj); + }); + + it('should throw on invalid JSON', () => { + expect(() => jsonlDecode('invalid json\n')).toThrow('Unexpected token'); + }); + + it('should handle Buffer input', () => { + const obj = { key: 'value', number: 42 }; + const jsonStr = `${JSON.stringify(obj)}\n`; + expect(jsonlDecode(Buffer.from(jsonStr))).toStrictEqual(obj); + }); + + it('should handle primitive JSON values', () => { + expect(jsonlDecode('"string"\n')).toBe('string'); + expect(jsonlDecode('42\n')).toBe(42); + expect(jsonlDecode('true\n')).toBe(true); + expect(jsonlDecode('null\n')).toBeNull(); + }); +}); + +describe('recoverJsonlFile', () => { + beforeEach(() => { + vol.fromJSON( + { + '/tmp': null, + }, + MEMFS_VOLUME, + ); + }); + + it('should recover JSONL file with single object', () => { + const filePath = '/tmp/recover-single.jsonl'; + const obj = { key: 'value', number: 42 }; + fs.writeFileSync(filePath, `${JSON.stringify(obj)}\n`); + + expect(recoverJsonlFile(filePath)).toStrictEqual({ + records: [obj], + errors: [], + partialTail: null, + }); + }); + + it('should recover JSONL file with multiple objects', () => { + const filePath = '/tmp/recover-multi.jsonl'; + const obj1 = { id: 1, name: 'first' }; + const obj2 = { id: 2, name: 'second' }; + fs.writeFileSync( + filePath, + `${JSON.stringify(obj1)}\n${JSON.stringify(obj2)}\n`, + ); + + expect(recoverJsonlFile(filePath)).toStrictEqual({ + records: [obj1, obj2], + errors: [], + partialTail: null, + }); + }); + + it('should handle JSON parsing errors', () => { + const filePath = '/tmp/recover-error.jsonl'; + fs.writeFileSync( + filePath, + '{"valid": "json"}\ninvalid json line\n{"id":3,"name":"Charlie","incomplete":\n', + ); + + const result = recoverJsonlFile(filePath); + expect(result.records).toStrictEqual([{ valid: 'json' }]); + expect(result.errors).toStrictEqual([ + expect.objectContaining({ line: 'invalid json line' }), + expect.objectContaining({ + line: '{"id":3,"name":"Charlie","incomplete":', + }), + ]); + expect(result.partialTail).toBe('{"id":3,"name":"Charlie","incomplete":'); + }); + + it('should support keepInvalid option', () => { + const filePath = '/tmp/recover-keep-invalid.jsonl'; + fs.writeFileSync(filePath, '{"valid": "json"}\ninvalid json\n'); + + const result = recoverJsonlFile(filePath, { keepInvalid: true }); + expect(result.records).toStrictEqual([ + { valid: 'json' }, + { __invalid: true, lineNo: 2, line: 'invalid json' }, + ]); + expect(result.errors).toHaveLength(1); + }); + + it('should handle empty files', () => { + const filePath = '/tmp/recover-empty.jsonl'; + fs.writeFileSync(filePath, ''); + + expect(recoverJsonlFile(filePath)).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + + it('should handle file read errors gracefully', () => { + expect(recoverJsonlFile('/nonexistent/file.jsonl')).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); +}); + +describe('JsonlFileSink', () => { + beforeEach(() => { + vol.fromJSON( + { + '/tmp': null, + }, + MEMFS_VOLUME, + ); + }); + + type JsonObj = { key: string; number: number }; + + it('should encode objects as JSON', () => { + const sink = new JsonlFileSink({ + filePath: '/tmp/jsonl-test.jsonl', + }); + const obj = { key: 'value', number: 42 }; + expect(sink.encode(obj)).toBe(`${JSON.stringify(obj)}\n`); + }); + + it('should decode JSON strings to objects', () => { + const sink = new JsonlFileSink({ + filePath: '/tmp/jsonl-test.jsonl', + }); + const obj = { key: 'value', number: 42 }; + const jsonStr = `${JSON.stringify(obj)}\n`; + expect(sink.decode(jsonStr)).toStrictEqual(obj); + }); + + it('should handle file operations with JSONL format', () => { + const filePath = '/tmp/jsonl-file-ops-test.jsonl'; + const sink = new JsonlFileSink({ filePath }); + sink.open(); + + const obj1 = { key: 'value', number: 42 }; + const obj2 = { key: 'value', number: 42 }; + sink.write(obj1); + sink.write(obj2); + sink.close(); + + const recovered = sink.recover(); + expect(recovered.records).toStrictEqual([obj1, obj2]); + }); + + it('repack() should recover records and write them to output path', () => { + const filePath = '/tmp/jsonl-repack-test.jsonl'; + const sink = new JsonlFileSink({ filePath }); + const records = [ + { key: 'value', number: 42 }, + { key: 'value', number: 42 }, + ]; + + fs.writeFileSync( + filePath, + `${records.map(record => JSON.stringify(record)).join('\n')}\n`, + ); + + const outputPath = '/tmp/jsonl-repack-output.jsonl'; + sink.repack(outputPath); + expect(fs.readFileSync(outputPath, 'utf8')).toBe( + `${JSON.stringify(records[0])}\n${JSON.stringify(records[1])}\n`, + ); + }); +}); diff --git a/packages/utils/src/lib/file-sink-text.int.test.ts b/packages/utils/src/lib/file-sink-text.int.test.ts new file mode 100644 index 000000000..19ea34fb0 --- /dev/null +++ b/packages/utils/src/lib/file-sink-text.int.test.ts @@ -0,0 +1,184 @@ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { teardownTestFolder } from '@code-pushup/test-utils'; +import { FileSink, stringRecover } from './file-sink-text.js'; + +describe('FileSink integration', () => { + const baseDir = path.join(os.tmpdir(), 'file-sink-text-int-tests'); + const testFile = path.join(baseDir, 'test-data.txt'); + + beforeAll(async () => { + await fs.promises.mkdir(baseDir, { recursive: true }); + }); + + beforeEach(async () => { + try { + await fs.promises.unlink(testFile); + } catch { + // File doesn't exist, which is fine + } + }); + + afterAll(async () => { + await teardownTestFolder(baseDir); + }); + + describe('file operations', () => { + const testData = ['line1', 'line2', 'line3']; + + it('should write and read text files', async () => { + const sink = new FileSink({ + filePath: testFile, + recover: () => stringRecover(testFile, (line: string) => line), + }); + + // Open and write data + sink.open(); + testData.forEach(item => sink.write(item)); + sink.close(); + + expect(fs.existsSync(testFile)).toBe(true); + const fileContent = fs.readFileSync(testFile, 'utf8'); + const lines = fileContent.trim().split('\n'); + expect(lines).toStrictEqual(testData); + + lines.forEach((line, index) => { + expect(line).toStrictEqual(testData[index]); + }); + }); + + it('should recover data from text files', async () => { + const content = `${testData.join('\n')}\n`; + fs.writeFileSync(testFile, content); + + expect(stringRecover(testFile, (line: string) => line)).toStrictEqual({ + records: testData, + errors: [], + partialTail: null, + }); + }); + + it('should handle text files with parse errors', async () => { + const mixedContent = 'valid\ninvalid\nanother\n'; + fs.writeFileSync(testFile, mixedContent); + + expect( + stringRecover(testFile, (line: string) => { + if (line === 'invalid') throw new Error('Invalid line'); + return line.toUpperCase(); + }), + ).toStrictEqual({ + records: ['VALID', 'ANOTHER'], + errors: [ + expect.objectContaining({ + lineNo: 2, + line: 'invalid', + error: expect.any(Error), + }), + ], + partialTail: 'invalid', + }); + }); + + it('should repack file with recovered data', async () => { + const sink = new FileSink({ + filePath: testFile, + recover: () => stringRecover(testFile, (line: string) => line), + }); + + // Write initial data + sink.open(); + testData.forEach(item => sink.write(item)); + sink.close(); + + // Repack to the same file + sink.repack(); + + // Verify the content is still correct + const fileContent = fs.readFileSync(testFile, 'utf8'); + const lines = fileContent + .trim() + .split('\n') + .filter(line => line.length > 0); + expect(lines).toStrictEqual(testData); + }); + + it('should repack file to different output path', async () => { + const outputPath = path.join(baseDir, 'repacked.txt'); + const sink = new FileSink({ + filePath: testFile, + recover: () => stringRecover(testFile, (line: string) => line), + }); + + // Write initial data + sink.open(); + testData.forEach(item => sink.write(item)); + sink.close(); + + // Repack to different file + sink.repack(outputPath); + + // Verify the original file is unchanged + expect(fs.existsSync(testFile)).toBe(true); + + // Verify the repacked file has correct content + expect(fs.existsSync(outputPath)).toBe(true); + const fileContent = fs.readFileSync(outputPath, 'utf8'); + const lines = fileContent + .trim() + .split('\n') + .filter(line => line.length > 0); + expect(lines).toStrictEqual(testData); + }); + + it('should call finalize function when provided', async () => { + let finalized = false; + const sink = new FileSink({ + filePath: testFile, + recover: () => stringRecover(testFile, (line: string) => line), + finalize: () => { + finalized = true; + }, + }); + + sink.finalize(); + expect(finalized).toBe(true); + }); + }); + + describe('edge cases', () => { + it('should handle empty files', async () => { + fs.writeFileSync(testFile, ''); + + expect(stringRecover(testFile, (line: string) => line)).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + + it('should handle files with only whitespace', async () => { + fs.writeFileSync(testFile, ' \n \n\t\n'); + + expect(stringRecover(testFile, (line: string) => line)).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + + it('should handle non-existent files', async () => { + const nonExistentFile = path.join(baseDir, 'does-not-exist.txt'); + + expect( + stringRecover(nonExistentFile, (line: string) => line), + ).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + }); +}); diff --git a/packages/utils/src/lib/file-sink-text.ts b/packages/utils/src/lib/file-sink-text.ts new file mode 100644 index 000000000..3cafacbe4 --- /dev/null +++ b/packages/utils/src/lib/file-sink-text.ts @@ -0,0 +1,147 @@ +import * as fs from 'node:fs'; +import { existsSync, mkdirSync } from 'node:fs'; +import path from 'node:path'; +import type { + RecoverOptions, + RecoverResult, + Recoverable, + Sink, +} from './sink-source.types.js'; + +export const stringDecode = (output: O): I => { + const str = Buffer.isBuffer(output) + ? output.toString('utf8') + : String(output); + return str as unknown as I; +}; + +export const stringEncode = (input: I): O => + `${typeof input === 'string' ? input : JSON.stringify(input)}\n` as O; + +export const stringRecover = function ( + filePath: string, + decode: (output: O) => I, + opts: RecoverOptions = {}, +): RecoverResult { + const records: I[] = []; + const errors: { lineNo: number; line: string; error: Error }[] = []; + let partialTail: string | null = null; + + try { + const content = fs.readFileSync(filePath, 'utf8'); + const lines = content.trim().split('\n'); + let lineNo = 0; + + for (const line of lines) { + lineNo++; + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + + try { + const record = decode(trimmedLine as O); + records.push(record); + } catch (error) { + const info = { lineNo, line, error: error as Error }; + errors.push(info); + + if (opts.keepInvalid) { + records.push({ __invalid: true, lineNo, line } as any); + } + + partialTail = line; + } + } + } catch { + return { records: [], errors: [], partialTail: null }; + } + + return { records, errors, partialTail }; +}; + +export type FileSinkOptions = { + filePath: string; + recover?: () => RecoverResult; + finalize?: () => void; +}; + +export type FileInput = Buffer | string; +export type FileOutput = Buffer | string; + +export class FileSink + implements Sink, Recoverable +{ + #fd: number | null = null; + options: FileSinkOptions; + + constructor(options: FileSinkOptions) { + this.options = options; + } + + isClosed(): boolean { + return this.#fd == null; + } + + encode(input: I): O { + return stringEncode(input as any); + } + + decode(output: O): I { + return stringDecode(output as any); + } + getFilePath(): string { + return this.options.filePath; + } + + open(withRepack: boolean = false): void { + const dir = path.dirname(this.options.filePath); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + if (withRepack) { + this.repack(this.options.filePath); + } + this.#fd = fs.openSync(this.options.filePath, 'a'); + } + + close(): void { + if (this.#fd == null) { + return; + } + fs.closeSync(this.#fd); + this.#fd = null; + } + + write(input: I): void { + if (this.#fd == null) { + return; + } // Silently ignore if not open + const encoded = this.encode(input); + try { + fs.writeSync(this.#fd, encoded as any); + } catch { + // Silently ignore write errors (e.g., EBADF in test environments with mocked fs) + } + } + + recover(): RecoverResult { + const dir = path.dirname(this.options.filePath); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + return this.options.recover!() as RecoverResult; + } + + repack(outputPath?: string): void { + const { records } = this.recover(); + fs.writeFileSync( + outputPath ?? this.getFilePath(), + records.map(this.encode).join('\n'), + ); + } + + finalize(): void { + this.options.finalize!(); + } +} diff --git a/packages/utils/src/lib/file-sink-text.unit.test.ts b/packages/utils/src/lib/file-sink-text.unit.test.ts new file mode 100644 index 000000000..f76cf13d4 --- /dev/null +++ b/packages/utils/src/lib/file-sink-text.unit.test.ts @@ -0,0 +1,295 @@ +import { vol } from 'memfs'; +import * as fs from 'node:fs'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { MEMFS_VOLUME } from '@code-pushup/test-utils'; +import { + FileSink, + type FileSinkOptions, + stringDecode, + stringEncode, + stringRecover, +} from './file-sink-text.js'; + +describe('stringEncode', () => { + it('stringEncode() should encode string input with newline', () => { + const str = 'test string'; + expect(stringEncode(str)).toBe(`${str}\n`); + }); + + it('stringEncode() should encode non-string input as JSON with newline', () => { + const obj = { key: 'value', number: 42 }; + expect(stringEncode(obj)).toBe(`${JSON.stringify(obj)}\n`); + }); + + it('stringEncode() should handle null input', () => { + expect(stringEncode(null)).toBe('null\n'); + }); + + it('stringEncode() should handle undefined input', () => { + expect(stringEncode(undefined)).toBe('undefined\n'); + }); +}); + +describe('stringDecode', () => { + it('stringDecode() should decode Buffer to string', () => { + const str = 'test content'; + expect(stringDecode(Buffer.from(str))).toBe(str); + }); + + it('stringDecode() should return string input as-is', () => { + const str = 'test string'; + expect(stringDecode(str)).toBe(str); + }); +}); + +describe('stringRecover', () => { + it('stringRecover() should recover records from valid file content', () => { + const filePath = '/tmp/stringRecover-test.txt'; + vol.fromJSON({ + [filePath]: 'line1\nline2\nline3\n', + }); + + expect(stringRecover(filePath, (line: string) => line)).toStrictEqual({ + records: ['line1', 'line2', 'line3'], + errors: [], + partialTail: null, + }); + }); + + it('stringRecover() should recover records and apply decode function', () => { + const filePath = '/tmp/stringRecover-test.txt'; + vol.fromJSON({ + [filePath]: 'line1\nline2\nline3\n', + }); + + expect( + stringRecover(filePath, (line: string) => line.toUpperCase()), + ).toStrictEqual({ + records: ['LINE1', 'LINE2', 'LINE3'], + errors: [], + partialTail: null, + }); + }); + + it('stringRecover() should skip empty lines', () => { + const filePath = '/tmp/stringRecover-empty-test.txt'; + vol.fromJSON({ + [filePath]: 'line1\n\nline2\n', + }); + + expect(stringRecover(filePath, (line: string) => line)).toStrictEqual({ + records: ['line1', 'line2'], + errors: [], + partialTail: null, + }); + }); + + it('stringRecover() should handle decode errors and continue processing', () => { + const filePath = '/tmp/stringRecover-error-test.txt'; + vol.fromJSON({ + [filePath]: 'valid\ninvalid\nanother', + }); + + expect( + stringRecover(filePath, (line: string) => { + if (line === 'invalid') throw new Error('Invalid line'); + return line.toUpperCase(); + }), + ).toStrictEqual({ + records: ['VALID', 'ANOTHER'], + errors: [ + { + lineNo: 2, + line: 'invalid', + error: expect.any(Error), + }, + ], + partialTail: 'invalid', + }); + }); + + it('stringRecover() should include invalid records when keepInvalid option is true', () => { + const filePath = '/tmp/stringRecover-invalid-test.txt'; + vol.fromJSON({ + [filePath]: 'valid\ninvalid\n', + }); + + expect( + stringRecover( + filePath, + (line: string) => { + if (line === 'invalid') throw new Error('Invalid line'); + return line.toUpperCase(); + }, + { keepInvalid: true }, + ), + ).toStrictEqual({ + records: ['VALID', { __invalid: true, lineNo: 2, line: 'invalid' }], + errors: [expect.any(Object)], + partialTail: 'invalid', + }); + }); + + it('stringRecover() should handle file read errors gracefully', () => { + expect( + stringRecover('/nonexistent/file.txt', (line: string) => line), + ).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); +}); + +describe('FileSink', () => { + it('constructor should create instance with options', () => { + const options: FileSinkOptions = { + filePath: '/tmp/test-file.txt', + recover: vi + .fn() + .mockReturnValue({ records: [], errors: [], partialTail: null }), + finalize: vi.fn(), + }; + expect(new FileSink(options).options).toBe(options); + }); + + it('getFilePath() should return the file path', () => { + const filePath = '/tmp/test-file.txt'; + const sink = new FileSink({ filePath }); + expect(sink.getFilePath()).toBe(filePath); + }); + + it('encode() should encode input using stringEncode', () => { + const sink = new FileSink({ filePath: '/tmp/test.txt' }); + const str = 'test input'; + expect(sink.encode(str)).toBe(`${str}\n`); + }); + + it('decode() should decode output using stringDecode', () => { + const sink = new FileSink({ filePath: '/tmp/test.txt' }); + const str = 'test output'; + expect(sink.decode(str)).toBe(str); + }); + + it('open() should handle directory creation and file opening', () => { + const sink = new FileSink({ filePath: '/tmp/test-file.txt' }); + sink.open(); + expect(fs.existsSync('/tmp/test-file.txt')).toBe(true); + }); + + it('open() should repack file when withRepack is true', () => { + const sink = new FileSink({ + filePath: '/tmp/test-file.txt', + recover: vi + .fn() + .mockReturnValue({ records: [], errors: [], partialTail: null }), + }); + const spy = vi.spyOn(sink, 'repack'); + sink.open(true); + expect(spy).toHaveBeenCalledWith('/tmp/test-file.txt'); + }); + + it('close() should close file descriptor if open', () => { + const sink = new FileSink({ filePath: '/tmp/test-file.txt' }); + sink.open(); + expect(() => sink.close()).not.toThrow(); + }); + + it('close() should do nothing if file descriptor is not open', () => { + const sink = new FileSink({ filePath: '/tmp/test-file.txt' }); + expect(() => sink.close()).not.toThrow(); + }); + + it('write() should write encoded input to file when sink is open', () => { + const sink = new FileSink({ filePath: '/tmp/write-open-unique-test.txt' }); + sink.open(); + const str = 'test data'; + sink.write(str); + expect(fs.readFileSync('/tmp/write-open-unique-test.txt', 'utf8')).toBe( + `${str}\n`, + ); + }); + + it('write() should silently ignore writes when file descriptor is not open', () => { + const sink = new FileSink({ filePath: '/tmp/write-test-closed.txt' }); + expect(() => sink.write('test data')).not.toThrow(); + }); + + it('write() should silently ignore write errors when fs.writeSync throws', () => { + const sink = new FileSink({ filePath: '/tmp/write-error-test.txt' }); + sink.open(); + + // Mock fs.writeSync to throw an error + const writeSyncSpy = vi.spyOn(fs, 'writeSync').mockImplementation(() => { + throw new Error('Write error'); + }); + + try { + // This should not throw despite the write error + expect(() => sink.write('test data')).not.toThrow(); + } finally { + // Restore original function + writeSyncSpy.mockRestore(); + sink.close(); + } + }); + + it('recover() should call the recover function from options', () => { + const mockRecover = vi + .fn() + .mockReturnValue({ records: ['test'], errors: [], partialTail: null }); + const sink = new FileSink({ + filePath: '/tmp/test-file.txt', + recover: mockRecover, + }); + expect(sink.recover()).toStrictEqual({ + records: ['test'], + errors: [], + partialTail: null, + }); + expect(mockRecover).toHaveBeenCalledWith(); + }); + + it('repack() should recover records and write them to output path', () => { + const mockRecover = vi.fn(); + const sink = new FileSink({ + filePath: '/tmp/test-file.txt', + recover: mockRecover, + }); + const records = ['record1', 'record2']; + mockRecover.mockReturnValue({ records, errors: [], partialTail: null }); + const outputPath = '/tmp/repack-output.txt'; + sink.repack(outputPath); + expect(mockRecover).toHaveBeenCalled(); + expect(fs.readFileSync(outputPath, 'utf8')).toBe('record1\n\nrecord2\n'); + }); + + it('finalize() should call the finalize function from options', () => { + const mockFinalize = vi.fn(); + const sink = new FileSink({ + filePath: '/tmp/test-file.txt', + finalize: mockFinalize, + }); + sink.finalize(); + expect(mockFinalize).toHaveBeenCalledTimes(1); + }); + + it('isClosed() should return true when sink is not opened', () => { + const sink = new FileSink({ filePath: '/tmp/test-file.txt' }); + expect(sink.isClosed()).toBe(true); + }); + + it('isClosed() should return false when sink is opened', () => { + const sink = new FileSink({ filePath: '/tmp/test-file.txt' }); + sink.open(); + expect(sink.isClosed()).toBe(false); + }); + + it('isClosed() should return true when sink is closed after being opened', () => { + const sink = new FileSink({ filePath: '/tmp/test-file.txt' }); + sink.open(); + expect(sink.isClosed()).toBe(false); + sink.close(); + expect(sink.isClosed()).toBe(true); + }); +}); diff --git a/packages/utils/src/lib/sink-source.types.ts b/packages/utils/src/lib/sink-source.types.ts new file mode 100644 index 000000000..4473026d0 --- /dev/null +++ b/packages/utils/src/lib/sink-source.types.ts @@ -0,0 +1,48 @@ +export type Encoder = { + encode: (input: I) => O; +}; + +export type Decoder = { + decode: (output: O) => I; +}; + +export type Sink = { + open: () => void; + write: (input: I) => void; + close: () => void; + isClosed: () => boolean; +} & Encoder; + +export type Buffered = { + flush: () => void; +}; +export type BufferedSink = {} & Sink & Buffered; + +export type Source = { + read?: () => O; + decode?: (input: I) => O; +}; + +export type Observer = { + subscribe: () => void; + unsubscribe: () => void; + isSubscribed: () => boolean; +}; + +export type Recoverable = { + recover: () => RecoverResult; + repack: (outputPath?: string) => void; + finalize: () => void; +}; + +export type RecoverResult = { + records: T[]; + errors: { lineNo: number; line: string; error: Error }[]; + partialTail: string | null; +}; + +export type RecoverOptions = { + keepInvalid?: boolean; +}; + +export type Output = {} & BufferedSink; From 756f8c0db48a28ec83126fea624ceb3acf60e6f6 Mon Sep 17 00:00:00 2001 From: John Doe Date: Fri, 9 Jan 2026 22:38:09 +0100 Subject: [PATCH 2/5] feat: add file sink classes --- ...nt.test.ts => file-sink-jsonl.int.test.ts} | 2 +- .../{file-sink-json.ts => file-sink-jsonl.ts} | 0 ...t.test.ts => file-sink-jsonl.unit.test.ts} | 28 ++++++++++++++++++- .../utils/src/lib/file-sink-text.unit.test.ts | 15 ++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) rename packages/utils/src/lib/{file-sink-json.int.test.ts => file-sink-jsonl.int.test.ts} (99%) rename packages/utils/src/lib/{file-sink-json.ts => file-sink-jsonl.ts} (100%) rename packages/utils/src/lib/{file-sink-json.unit.test.ts => file-sink-jsonl.unit.test.ts} (89%) diff --git a/packages/utils/src/lib/file-sink-json.int.test.ts b/packages/utils/src/lib/file-sink-jsonl.int.test.ts similarity index 99% rename from packages/utils/src/lib/file-sink-json.int.test.ts rename to packages/utils/src/lib/file-sink-jsonl.int.test.ts index c331d8d0a..e0f57bbaa 100644 --- a/packages/utils/src/lib/file-sink-json.int.test.ts +++ b/packages/utils/src/lib/file-sink-jsonl.int.test.ts @@ -3,7 +3,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { teardownTestFolder } from '@code-pushup/test-utils'; -import { JsonlFileSink, recoverJsonlFile } from './file-sink-json.js'; +import { JsonlFileSink, recoverJsonlFile } from './file-sink-jsonl.js'; describe('JsonlFileSink integration', () => { const baseDir = path.join(os.tmpdir(), 'file-sink-json-int-tests'); diff --git a/packages/utils/src/lib/file-sink-json.ts b/packages/utils/src/lib/file-sink-jsonl.ts similarity index 100% rename from packages/utils/src/lib/file-sink-json.ts rename to packages/utils/src/lib/file-sink-jsonl.ts diff --git a/packages/utils/src/lib/file-sink-json.unit.test.ts b/packages/utils/src/lib/file-sink-jsonl.unit.test.ts similarity index 89% rename from packages/utils/src/lib/file-sink-json.unit.test.ts rename to packages/utils/src/lib/file-sink-jsonl.unit.test.ts index a920c8cbe..75f981cb0 100644 --- a/packages/utils/src/lib/file-sink-json.unit.test.ts +++ b/packages/utils/src/lib/file-sink-jsonl.unit.test.ts @@ -7,7 +7,7 @@ import { jsonlDecode, jsonlEncode, recoverJsonlFile, -} from './file-sink-json.js'; +} from './file-sink-jsonl.js'; describe('jsonlEncode', () => { it('should encode object to JSON string', () => { @@ -207,10 +207,36 @@ describe('JsonlFileSink', () => { `${records.map(record => JSON.stringify(record)).join('\n')}\n`, ); + sink.repack(); + expect(fs.readFileSync(filePath, 'utf8')).toBe( + `${JSON.stringify(records[0])}\n${JSON.stringify(records[1])}\n`, + ); + }); + + it('repack() should accept output path', () => { + const filePath = '/tmp/jsonl-repack-test.jsonl'; + const sink = new JsonlFileSink({ filePath }); + const records = [ + { key: 'value', number: 42 }, + { key: 'value', number: 42 }, + ]; + + fs.writeFileSync( + filePath, + `${records.map(record => JSON.stringify(record)).join('\n')}\n`, + ); + const outputPath = '/tmp/jsonl-repack-output.jsonl'; sink.repack(outputPath); expect(fs.readFileSync(outputPath, 'utf8')).toBe( `${JSON.stringify(records[0])}\n${JSON.stringify(records[1])}\n`, ); }); + + it('should do nothing on finalize()', () => { + const sink = new JsonlFileSink({ + filePath: '/tmp/jsonl-finalize-test.jsonl', + }); + expect(() => sink.finalize()).not.toThrow(); + }); }); diff --git a/packages/utils/src/lib/file-sink-text.unit.test.ts b/packages/utils/src/lib/file-sink-text.unit.test.ts index f76cf13d4..33cc9ad0e 100644 --- a/packages/utils/src/lib/file-sink-text.unit.test.ts +++ b/packages/utils/src/lib/file-sink-text.unit.test.ts @@ -251,6 +251,21 @@ describe('FileSink', () => { }); it('repack() should recover records and write them to output path', () => { + const mockRecover = vi.fn(); + const filePath = '/tmp/test-file.txt'; + const sink = new FileSink({ + filePath, + recover: mockRecover, + }); + const records = ['record1', 'record2']; + mockRecover.mockReturnValue({ records, errors: [], partialTail: null }); + + sink.repack(); + expect(mockRecover).toHaveBeenCalled(); + expect(fs.readFileSync(filePath, 'utf8')).toBe('record1\n\nrecord2\n'); + }); + + it('repack() should accept output path', () => { const mockRecover = vi.fn(); const sink = new FileSink({ filePath: '/tmp/test-file.txt', From b0c9cc4d9238a10ce979dbde284b9f30329bf4c2 Mon Sep 17 00:00:00 2001 From: John Doe Date: Tue, 13 Jan 2026 00:01:27 +0100 Subject: [PATCH 3/5] refactor: add trace json file --- .../utils/src/lib/file-sink-json-trace.ts | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 packages/utils/src/lib/file-sink-json-trace.ts diff --git a/packages/utils/src/lib/file-sink-json-trace.ts b/packages/utils/src/lib/file-sink-json-trace.ts new file mode 100644 index 000000000..7933d318c --- /dev/null +++ b/packages/utils/src/lib/file-sink-json-trace.ts @@ -0,0 +1,167 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { JsonlFileSink, recoverJsonlFile } from './file-sink-jsonl.js'; +import { getCompleteEvent, getStartTracing } from './trace-file-utils.js'; +import type { + InstantEvent, + SpanEvent, + TraceEvent, + TraceEventRaw, + UserTimingDetail, +} from './trace-file.type.js'; + +const tryJson = (v: unknown): T | unknown => { + if (typeof v !== 'string') return v; + try { + return JSON.parse(v) as T; + } catch { + return v; + } +}; + +const toJson = (v: unknown): unknown => { + if (v === undefined) return undefined; + try { + return JSON.stringify(v); + } catch { + return v; + } +}; + +export function decodeTraceEvent({ args, ...rest }: TraceEventRaw): TraceEvent { + if (!args) return rest as TraceEvent; + + const out: any = { ...args }; + if ('detail' in out) out.detail = tryJson(out.detail); + if (out.data?.detail) + out.data.detail = tryJson(out.data.detail); + + return { ...rest, args: out } as TraceEvent; +} + +export function encodeTraceEvent({ args, ...rest }: TraceEvent): TraceEventRaw { + if (!args) return rest as TraceEventRaw; + + const out: any = { ...args }; + if ('detail' in out) out.detail = toJson(out.detail); + if (out.data?.detail) out.data.detail = toJson(out.data.detail); + + return { ...rest, args: out } as TraceEventRaw; +} + +function getTraceMetadata( + startDate?: Date, + metadata?: Record, +) { + return { + source: 'DevTools', + startTime: startDate?.toISOString() ?? new Date().toISOString(), + hardwareConcurrency: 1, + dataOrigin: 'TraceEvents', + ...metadata, + }; +} + +function createTraceFileContent( + traceEventsContent: string, + startDate?: Date, + metadata?: Record, +): string { + return `{ + "metadata": ${JSON.stringify(getTraceMetadata(startDate, metadata))}, + "traceEvents": [ +${traceEventsContent} + ] +}`; +} + +function finalizeTraceFile( + events: (SpanEvent | InstantEvent)[], + outputPath: string, + metadata?: Record, +): void { + const { writeFileSync } = fs; + + const sortedEvents = events.sort((a, b) => a.ts - b.ts); + const first = sortedEvents[0]; + const last = sortedEvents[sortedEvents.length - 1]; + + // Use performance.now() as fallback when no events exist + const fallbackTs = performance.now(); + const firstTs = first?.ts ?? fallbackTs; + const lastTs = last?.ts ?? fallbackTs; + + // Add margins for readability + const tsMargin = 1000; + const startTs = firstTs - tsMargin; + const endTs = lastTs + tsMargin; + const startDate = new Date().toISOString(); + + const traceEventsJson = [ + // Preamble + encodeTraceEvent( + getStartTracing({ + ts: startTs, + url: outputPath, + }), + ), + encodeTraceEvent( + getCompleteEvent({ + ts: startTs, + dur: 20, + }), + ), + // Events + ...events.map(encodeTraceEvent), + encodeTraceEvent( + getCompleteEvent({ + ts: endTs, + dur: 20, + }), + ), + ].join(',\n'); + + const jsonOutput = createTraceFileContent( + traceEventsJson, + new Date(), + metadata, + ); + writeFileSync(outputPath, jsonOutput, 'utf8'); +} + +export interface TraceFileSinkOptions { + filename: string; + directory?: string; + metadata?: Record; +} + +export class TraceFileSink extends JsonlFileSink { + readonly #filePath: string; + readonly #getFilePathForExt: (ext: 'json' | 'jsonl') => string; + readonly #metadata: Record | undefined; + + constructor(opts: TraceFileSinkOptions) { + const { filename, directory = '.', metadata } = opts; + + const traceJsonlPath = path.join(directory, `${filename}.jsonl`); + + super({ + filePath: traceJsonlPath, + recover: () => recoverJsonlFile(traceJsonlPath), + }); + + this.#metadata = metadata; + this.#filePath = path.join(directory, `${filename}.json`); + this.#getFilePathForExt = (ext: 'json' | 'jsonl') => + path.join(directory, `${filename}.${ext}`); + } + + override finalize(): void { + finalizeTraceFile(this.recover().records, this.#filePath, this.#metadata); + } + + getFilePathForExt(ext: 'json' | 'jsonl'): string { + return this.#getFilePathForExt(ext); + } +} From 1f6e32628e5a2aeffea6c5c560acb60868abe324 Mon Sep 17 00:00:00 2001 From: John Doe Date: Wed, 14 Jan 2026 01:05:15 +0100 Subject: [PATCH 4/5] refactor: wip --- packages/utils/src/lib/file-sink-text.ts | 37 ++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/packages/utils/src/lib/file-sink-text.ts b/packages/utils/src/lib/file-sink-text.ts index 3cafacbe4..050188e58 100644 --- a/packages/utils/src/lib/file-sink-text.ts +++ b/packages/utils/src/lib/file-sink-text.ts @@ -1,6 +1,7 @@ import * as fs from 'node:fs'; import { existsSync, mkdirSync } from 'node:fs'; import path from 'node:path'; +import { PROFILER_FILE_BASE_NAME, PROFILER_OUT_DIR } from './profiler'; import type { RecoverOptions, RecoverResult, @@ -60,6 +61,42 @@ export const stringRecover = function ( return { records, errors, partialTail }; }; +export type FileNameOptions = { + fileBaseName: string; + outDir: string; + fileName?: string; +}; + +export function getFilenameParts(options: FileNameOptions): { + outDir: string; + fileName: string; +} { + const { fileName, fileBaseName, outDir } = options; + + if (fileName) { + return { + outDir, + fileName, + }; + } + + const baseName = fileBaseName; + const DATE_LENGTH = 10; + const TIME_SEGMENTS = 3; + const COLON_LENGTH = 1; + const TOTAL_TIME_LENGTH = + TIME_SEGMENTS * 2 + (TIME_SEGMENTS - 1) * COLON_LENGTH; // HH:MM:SS = 8 chars + const id = new Date() + .toISOString() + .slice(0, DATE_LENGTH + TOTAL_TIME_LENGTH) + .replace(/:/g, '-'); + + return { + outDir, + fileName: `${baseName}.${id}`, + }; +} + export type FileSinkOptions = { filePath: string; recover?: () => RecoverResult; From 6bcb73b7e6d9c967cef339d371e56a99df0fc4c8 Mon Sep 17 00:00:00 2001 From: John Doe Date: Wed, 14 Jan 2026 02:23:33 +0100 Subject: [PATCH 5/5] refactor: wip --- .../src/lib/file-sink-json-trace.int.test.ts | 224 ++++++++++++ .../utils/src/lib/file-sink-json-trace.ts | 88 +++-- .../src/lib/file-sink-json-trace.unit.test.ts | 335 ++++++++++++++++++ 3 files changed, 613 insertions(+), 34 deletions(-) create mode 100644 packages/utils/src/lib/file-sink-json-trace.int.test.ts create mode 100644 packages/utils/src/lib/file-sink-json-trace.unit.test.ts diff --git a/packages/utils/src/lib/file-sink-json-trace.int.test.ts b/packages/utils/src/lib/file-sink-json-trace.int.test.ts new file mode 100644 index 000000000..e71bb90d5 --- /dev/null +++ b/packages/utils/src/lib/file-sink-json-trace.int.test.ts @@ -0,0 +1,224 @@ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { teardownTestFolder } from '@code-pushup/test-utils'; +import { TraceFileSink } from './file-sink-json-trace.js'; +import type { TraceEvent } from './trace-file.type'; + +describe('TraceFileSink integration', () => { + const baseDir = path.join(os.tmpdir(), 'file-sink-json-trace-int-tests'); + const traceJsonPath = path.join(baseDir, 'test-data.json'); + const traceJsonlPath = path.join(baseDir, 'test-data.jsonl'); + + beforeAll(async () => { + await fs.promises.mkdir(baseDir, { recursive: true }); + }); + + beforeEach(async () => { + try { + await fs.promises.unlink(traceJsonPath); + } catch { + // File doesn't exist, which is fine + } + try { + await fs.promises.unlink(traceJsonlPath); + } catch { + // File doesn't exist, which is fine + } + }); + + afterAll(async () => { + await teardownTestFolder(baseDir); + }); + + describe('file operations', () => { + const testEvents: TraceEvent[] = [ + { name: 'navigationStart', ts: 100, ph: 'I', cat: 'blink.user_timing' }, + { + name: 'loadEventStart', + ts: 200, + ph: 'I', + cat: 'blink.user_timing', + args: { data: { url: 'https://example.com' } }, + }, + { + name: 'loadEventEnd', + ts: 250, + ph: 'I', + cat: 'blink.user_timing', + args: { detail: { duration: 50 } }, + }, + ]; + + it('should write and read trace events', async () => { + const sink = new TraceFileSink({ + filename: 'test-data', + directory: baseDir, + }); + + // Open and write data + sink.open(); + testEvents.forEach(event => sink.write(event as any)); + sink.finalize(); + + expect(fs.existsSync(traceJsonPath)).toBe(true); + expect(fs.existsSync(traceJsonlPath)).toBe(true); + + const jsonContent = fs.readFileSync(traceJsonPath, 'utf8'); + const traceData = JSON.parse(jsonContent); + + expect(traceData.metadata.source).toBe('DevTools'); + expect(traceData.metadata.dataOrigin).toBe('TraceEvents'); + expect(Array.isArray(traceData.traceEvents)).toBe(true); + + // Should have preamble events + user events + complete event + expect(traceData.traceEvents.length).toBeGreaterThan(testEvents.length); + + // Check that our events are included + const userEvents = traceData.traceEvents.filter((e: any) => + testEvents.some(testEvent => testEvent.name === e.name), + ); + expect(userEvents).toHaveLength(testEvents.length); + }); + + it('should recover events from JSONL file', async () => { + const sink = new TraceFileSink({ + filename: 'test-data', + directory: baseDir, + }); + sink.open(); + testEvents.forEach(event => sink.write(event as any)); + sink.close(); + + const recovered = sink.recover(); + expect(recovered.records).toStrictEqual(testEvents); + expect(recovered.errors).toStrictEqual([]); + expect(recovered.partialTail).toBeNull(); + }); + + it('should handle empty trace files', async () => { + const sink = new TraceFileSink({ + filename: 'empty-test', + directory: baseDir, + }); + sink.open(); + sink.finalize(); + + const emptyJsonPath = path.join(baseDir, 'empty-test.json'); + expect(fs.existsSync(emptyJsonPath)).toBe(true); + + const jsonContent = fs.readFileSync(emptyJsonPath, 'utf8'); + const traceData = JSON.parse(jsonContent); + + expect(traceData.metadata.source).toBe('DevTools'); + // Should have at least preamble and complete events + expect(traceData.traceEvents.length).toBeGreaterThanOrEqual(2); + }); + + it('should handle metadata in trace files', async () => { + const metadata = { + version: '1.0.0', + environment: 'test', + customData: { key: 'value' }, + }; + + const sink = new TraceFileSink({ + filename: 'metadata-test', + directory: baseDir, + metadata, + }); + sink.open(); + sink.write({ name: 'test-event', ts: 100, ph: 'I' } as any); + sink.finalize(); + + const metadataJsonPath = path.join(baseDir, 'metadata-test.json'); + const jsonContent = fs.readFileSync(metadataJsonPath, 'utf8'); + const traceData = JSON.parse(jsonContent); + + expect(traceData.metadata.version).toBe('1.0.0'); + expect(traceData.metadata.environment).toBe('test'); + expect(traceData.metadata.customData).toStrictEqual({ key: 'value' }); + expect(traceData.metadata.source).toBe('DevTools'); + }); + + describe('edge cases', () => { + it('should handle single event traces', async () => { + const singleEvent: TraceEvent = { + name: 'singleEvent', + ts: 123, + ph: 'I', + cat: 'test', + }; + + const sink = new TraceFileSink({ + filename: 'single-event-test', + directory: baseDir, + }); + sink.open(); + sink.write(singleEvent as any); + sink.finalize(); + + const singleJsonPath = path.join(baseDir, 'single-event-test.json'); + const jsonContent = fs.readFileSync(singleJsonPath, 'utf8'); + const traceData = JSON.parse(jsonContent); + + expect( + traceData.traceEvents.some((e: any) => e.name === 'singleEvent'), + ).toBe(true); + }); + + it('should handle events with complex args', async () => { + const complexEvent: TraceEvent = { + name: 'complexEvent', + ts: 456, + ph: 'X', + cat: 'test', + args: { + detail: { nested: { data: [1, 2, 3] } }, + data: { url: 'https://example.com', size: 1024 }, + }, + }; + + const sink = new TraceFileSink({ + filename: 'complex-args-test', + directory: baseDir, + }); + sink.open(); + sink.write(complexEvent as any); + sink.finalize(); + + const complexJsonPath = path.join(baseDir, 'complex-args-test.json'); + const jsonContent = fs.readFileSync(complexJsonPath, 'utf8'); + const traceData = JSON.parse(jsonContent); + + const eventInTrace = traceData.traceEvents.find( + (e: any) => e.name === 'complexEvent', + ); + expect(eventInTrace).toBeDefined(); + expect(eventInTrace.args.detail).toStrictEqual( + '{"nested":{"data":[1,2,3]}}', + ); + expect(eventInTrace.args.data.url).toBe('https://example.com'); + }); + + it('should handle non-existent directories gracefully', async () => { + const nonExistentDir = path.join(baseDir, 'non-existent'); + const sink = new TraceFileSink({ + filename: 'non-existent-dir-test', + directory: nonExistentDir, + }); + + sink.open(); + sink.write({ name: 'test', ts: 100, ph: 'I' } as any); + sink.finalize(); + + const jsonPath = path.join( + nonExistentDir, + 'non-existent-dir-test.json', + ); + expect(fs.existsSync(jsonPath)).toBe(true); + }); + }); + }); +}); diff --git a/packages/utils/src/lib/file-sink-json-trace.ts b/packages/utils/src/lib/file-sink-json-trace.ts index 7933d318c..f35895303 100644 --- a/packages/utils/src/lib/file-sink-json-trace.ts +++ b/packages/utils/src/lib/file-sink-json-trace.ts @@ -1,7 +1,12 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { performance } from 'node:perf_hooks'; -import { JsonlFileSink, recoverJsonlFile } from './file-sink-jsonl.js'; +import { + JsonlFileSink, + jsonlDecode, + jsonlEncode, + recoverJsonlFile, +} from './file-sink-jsonl.js'; import { getCompleteEvent, getStartTracing } from './trace-file-utils.js'; import type { InstantEvent, @@ -11,46 +16,60 @@ import type { UserTimingDetail, } from './trace-file.type.js'; -const tryJson = (v: unknown): T | unknown => { - if (typeof v !== 'string') return v; - try { - return JSON.parse(v) as T; - } catch { - return v; +export function decodeDetail(target: UserTimingDetail): UserTimingDetail { + if (typeof target.detail === 'string') { + return { ...target, detail: jsonlDecode(target.detail) }; } -}; + return target; +} -const toJson = (v: unknown): unknown => { - if (v === undefined) return undefined; - try { - return JSON.stringify(v); - } catch { - return v; +export function encodeDetail(target: UserTimingDetail): UserTimingDetail { + if (target.detail && typeof target.detail === 'object') { + return { + ...target, + detail: jsonlEncode(target.detail as UserTimingDetail), + }; } -}; + return target; +} export function decodeTraceEvent({ args, ...rest }: TraceEventRaw): TraceEvent { if (!args) return rest as TraceEvent; - const out: any = { ...args }; - if ('detail' in out) out.detail = tryJson(out.detail); - if (out.data?.detail) - out.data.detail = tryJson(out.data.detail); + const out: UserTimingDetail = { ...args }; + const processedOut = decodeDetail(out); - return { ...rest, args: out } as TraceEvent; + return { + ...rest, + args: + out.data && typeof out.data === 'object' + ? { + ...processedOut, + data: decodeDetail(out.data as UserTimingDetail), + } + : processedOut, + }; } export function encodeTraceEvent({ args, ...rest }: TraceEvent): TraceEventRaw { if (!args) return rest as TraceEventRaw; - const out: any = { ...args }; - if ('detail' in out) out.detail = toJson(out.detail); - if (out.data?.detail) out.data.detail = toJson(out.data.detail); + const out: UserTimingDetail = { ...args }; + const processedOut = encodeDetail(out); - return { ...rest, args: out } as TraceEventRaw; + return { + ...rest, + args: + out.data && typeof out.data === 'object' + ? { + ...processedOut, + data: encodeDetail(out.data as UserTimingDetail), + } + : processedOut, + }; } -function getTraceMetadata( +export function getTraceMetadata( startDate?: Date, metadata?: Record, ) { @@ -76,30 +95,30 @@ ${traceEventsContent} }`; } -function finalizeTraceFile( +export function finalizeTraceFile( events: (SpanEvent | InstantEvent)[], outputPath: string, metadata?: Record, ): void { const { writeFileSync } = fs; + if (events.length === 0) { + return; + } + const sortedEvents = events.sort((a, b) => a.ts - b.ts); const first = sortedEvents[0]; const last = sortedEvents[sortedEvents.length - 1]; - // Use performance.now() as fallback when no events exist const fallbackTs = performance.now(); const firstTs = first?.ts ?? fallbackTs; const lastTs = last?.ts ?? fallbackTs; - // Add margins for readability const tsMargin = 1000; const startTs = firstTs - tsMargin; const endTs = lastTs + tsMargin; - const startDate = new Date().toISOString(); const traceEventsJson = [ - // Preamble encodeTraceEvent( getStartTracing({ ts: startTs, @@ -112,7 +131,6 @@ function finalizeTraceFile( dur: 20, }), ), - // Events ...events.map(encodeTraceEvent), encodeTraceEvent( getCompleteEvent({ @@ -120,7 +138,9 @@ function finalizeTraceFile( dur: 20, }), ), - ].join(',\n'); + ] + .map(event => JSON.stringify(event)) + .join(',\n'); const jsonOutput = createTraceFileContent( traceEventsJson, @@ -130,11 +150,11 @@ function finalizeTraceFile( writeFileSync(outputPath, jsonOutput, 'utf8'); } -export interface TraceFileSinkOptions { +export type TraceFileSinkOptions = { filename: string; directory?: string; metadata?: Record; -} +}; export class TraceFileSink extends JsonlFileSink { readonly #filePath: string; diff --git a/packages/utils/src/lib/file-sink-json-trace.unit.test.ts b/packages/utils/src/lib/file-sink-json-trace.unit.test.ts new file mode 100644 index 000000000..162f5f048 --- /dev/null +++ b/packages/utils/src/lib/file-sink-json-trace.unit.test.ts @@ -0,0 +1,335 @@ +import { vol } from 'memfs'; +import * as fs from 'node:fs'; +import { beforeEach, describe, expect, it } from 'vitest'; +import { MEMFS_VOLUME } from '@code-pushup/test-utils'; +import { + TraceFileSink, + decodeTraceEvent, + encodeTraceEvent, + finalizeTraceFile, + getTraceMetadata, +} from './file-sink-json-trace.js'; +import type { + InstantEvent, + TraceEvent, + TraceEventRaw, +} from './trace-file.type'; + +describe('decodeTraceEvent', () => { + it('should return event without args if no args present', () => { + const event: TraceEventRaw = { name: 'test', ts: 123 }; + expect(decodeTraceEvent(event)).toStrictEqual(event); + }); + + it('should decode args with detail property', () => { + const event: TraceEventRaw = { + name: 'test', + ts: 123, + args: { detail: '{"key":"value"}' }, + }; + expect(decodeTraceEvent(event)).toStrictEqual({ + name: 'test', + ts: 123, + args: { detail: { key: 'value' } }, + }); + }); + + it('should decode nested data.detail property', () => { + const event: TraceEventRaw = { + name: 'test', + ts: 123, + args: { data: { detail: '{"nested":"value"}' } }, + }; + expect(decodeTraceEvent(event)).toStrictEqual({ + name: 'test', + ts: 123, + args: { data: { detail: { nested: 'value' } } }, + }); + }); + + it('should handle invalid JSON in detail', () => { + const event: TraceEventRaw = { + name: 'test', + ts: 123, + args: { detail: 'invalid json' }, + }; + expect(() => decodeTraceEvent(event)).toThrow('Unexpected token'); + }); +}); + +describe('encodeTraceEvent', () => { + it('should return event without args if no args present', () => { + const event: TraceEventRaw = { name: 'test', ts: 123 }; + expect(encodeTraceEvent(event)).toStrictEqual(event); + }); + + it('should encode args with detail property', () => { + const event: TraceEventRaw = { + name: 'test', + ts: 123, + args: { detail: { key: 'value' } }, + }; + expect(encodeTraceEvent(event)).toStrictEqual({ + name: 'test', + ts: 123, + args: { detail: '{"key":"value"}' }, + }); + }); + + it('should encode nested data.detail property', () => { + const event: TraceEventRaw = { + name: 'test', + ts: 123, + args: { data: { detail: { nested: 'value' } } }, + }; + expect(encodeTraceEvent(event)).toStrictEqual({ + name: 'test', + ts: 123, + args: { data: { detail: '{"nested":"value"}' } }, + }); + }); + + it('should handle non-serializable detail', () => { + const circular: any = {}; + circular.self = circular; + const event: TraceEventRaw = { + name: 'test', + ts: 123, + args: { detail: circular }, + }; + expect(() => encodeTraceEvent(event)).toThrow( + 'Converting circular structure to JSON', + ); + }); +}); + +describe('finalizeTraceFile', () => { + beforeEach(() => { + vol.fromJSON( + { + '/tmp': null, + }, + MEMFS_VOLUME, + ); + }); + + it('should create trace file with events', () => { + const events: TraceEvent[] = [ + { name: 'event1', ts: 100, ph: 'I' }, + { name: 'event2', ts: 200, ph: 'X', args: { dur: 50 } }, + ]; + const outputPath = '/tmp/test-trace.json'; + + finalizeTraceFile(events as any, outputPath); + + expect(fs.existsSync(outputPath)).toBe(true); + const content = JSON.parse(fs.readFileSync(outputPath, 'utf8')); + expect(content.metadata.source).toBe('DevTools'); + expect(content.traceEvents).toHaveLength(5); // preamble (start + complete) + events + complete + }); + + it('should handle empty events array', () => { + const events: TraceEvent[] = []; + const outputPath = '/tmp/empty-trace.json'; + + finalizeTraceFile(events as any, outputPath); + + expect(fs.existsSync(outputPath)).toBe(true); + const content = JSON.parse(fs.readFileSync(outputPath, 'utf8')); + expect(content.traceEvents).toHaveLength(3); // preamble (start + complete) + end complete + }); + + it('should sort events by timestamp', () => { + const events: TraceEvent[] = [ + { name: 'event2', ts: 200, ph: 'I' }, + { name: 'event1', ts: 100, ph: 'I' }, + ]; + const outputPath = '/tmp/sorted-trace.json'; + + finalizeTraceFile(events as any, outputPath); + + const content = JSON.parse(fs.readFileSync(outputPath, 'utf8')); + const eventNames = content.traceEvents + .filter((e: any) => e.name.startsWith('event')) + .map((e: any) => e.name); + expect(eventNames).toStrictEqual(['event1', 'event2']); + }); +}); + +describe('TraceFileSink', () => { + beforeEach(() => { + vol.fromJSON( + { + '/tmp': null, + }, + MEMFS_VOLUME, + ); + }); + + it('should create trace file sink with default options', () => { + const sink = new TraceFileSink({ filename: 'test' }); + expect(sink.getFilePathForExt('json')).toBe('test.json'); + expect(sink.getFilePathForExt('jsonl')).toBe('test.jsonl'); + }); + + it('should create trace file sink with custom directory', () => { + const sink = new TraceFileSink({ + filename: 'test', + directory: '/tmp/custom', + }); + expect(sink.getFilePathForExt('json')).toBe('/tmp/custom/test.json'); + expect(sink.getFilePathForExt('jsonl')).toBe('/tmp/custom/test.jsonl'); + }); + + it('should handle file operations with trace events', () => { + const sink = new TraceFileSink({ + filename: 'trace-test', + directory: '/tmp', + }); + sink.open(); + + const event1: InstantEvent = { name: 'mark1', ts: 100, ph: 'I' }; + const event2: InstantEvent = { name: 'mark2', ts: 200, ph: 'I' }; + sink.write(event1); + sink.write(event2); + sink.close(); + + expect(fs.existsSync('/tmp/trace-test.jsonl')).toBe(true); + expect(fs.existsSync('/tmp/trace-test.json')).toBe(false); + + const recovered = sink.recover(); + expect(recovered.records).toStrictEqual([event1, event2]); + }); + + it('should create trace file on finalize', () => { + const sink = new TraceFileSink({ + filename: 'finalize-test', + directory: '/tmp', + }); + sink.open(); + + const event: InstantEvent = { name: 'test-event', ts: 150, ph: 'I' }; + sink.write(event); + sink.finalize(); + + expect(fs.existsSync('/tmp/finalize-test.json')).toBe(true); + const content = JSON.parse( + fs.readFileSync('/tmp/finalize-test.json', 'utf8'), + ); + expect(content.metadata.source).toBe('DevTools'); + expect(content.traceEvents.some((e: any) => e.name === 'test-event')).toBe( + true, + ); + }); + + it('should handle metadata in finalize', () => { + const metadata = { customField: 'value', version: '1.0' }; + const sink = new TraceFileSink({ + filename: 'metadata-test', + directory: '/tmp', + metadata, + }); + sink.open(); + sink.write({ name: 'event', ts: 100, ph: 'I' }); + sink.finalize(); + + const content = JSON.parse( + fs.readFileSync('/tmp/metadata-test.json', 'utf8'), + ); + expect(content.metadata.customField).toBe('value'); + expect(content.metadata.version).toBe('1.0'); + }); + + it('should do nothing on finalize when no events written', () => { + const sink = new TraceFileSink({ + filename: 'empty-test', + directory: '/tmp', + }); + sink.open(); + sink.finalize(); + + expect(fs.existsSync('/tmp/empty-test.json')).toBe(true); + const content = JSON.parse(fs.readFileSync('/tmp/empty-test.json', 'utf8')); + expect(content.traceEvents).toHaveLength(3); // preamble (start + complete) + end complete + }); +}); + +describe('getTraceMetadata', () => { + it('should use provided startDate when given', () => { + const startDate = new Date('2023-01-15T10:30:00.000Z'); + const metadata = { customField: 'value' }; + + const result = getTraceMetadata(startDate, metadata); + + expect(result).toStrictEqual({ + source: 'DevTools', + startTime: '2023-01-15T10:30:00.000Z', + hardwareConcurrency: 1, + dataOrigin: 'TraceEvents', + customField: 'value', + }); + }); + + it('should use current date when startDate is undefined', () => { + const beforeTest = new Date(); + const metadata = { version: '1.0' }; + + const result = getTraceMetadata(undefined, metadata); + + const afterTest = new Date(); + expect(result.source).toBe('DevTools'); + expect(result.hardwareConcurrency).toBe(1); + expect(result.dataOrigin).toBe('TraceEvents'); + + // Verify startTime is a valid ISO string between test execution + const startTime = new Date(result.startTime); + expect(startTime.getTime()).toBeGreaterThanOrEqual(beforeTest.getTime()); + expect(startTime.getTime()).toBeLessThanOrEqual(afterTest.getTime()); + }); + + it('should use current date when startDate is null', () => { + const beforeTest = new Date(); + const metadata = { environment: 'test' }; + + const result = getTraceMetadata(undefined, metadata); + + const afterTest = new Date(); + expect(result.source).toBe('DevTools'); + expect(result.hardwareConcurrency).toBe(1); + expect(result.dataOrigin).toBe('TraceEvents'); + + // Verify startTime is a valid ISO string between test execution + const startTime = new Date(result.startTime); + expect(startTime.getTime()).toBeGreaterThanOrEqual(beforeTest.getTime()); + expect(startTime.getTime()).toBeLessThanOrEqual(afterTest.getTime()); + }); + + it('should handle empty metadata', () => { + const startDate = new Date('2023-12-25T00:00:00.000Z'); + + const result = getTraceMetadata(startDate); + + expect(result).toStrictEqual({ + source: 'DevTools', + startTime: '2023-12-25T00:00:00.000Z', + hardwareConcurrency: 1, + dataOrigin: 'TraceEvents', + }); + }); + + it('should handle both startDate and metadata undefined', () => { + const beforeTest = new Date(); + + const result = getTraceMetadata(); + + const afterTest = new Date(); + expect(result.source).toBe('DevTools'); + expect(result.hardwareConcurrency).toBe(1); + expect(result.dataOrigin).toBe('TraceEvents'); + + // Verify startTime is a valid ISO string between test execution + const startTime = new Date(result.startTime); + expect(startTime.getTime()).toBeGreaterThanOrEqual(beforeTest.getTime()); + expect(startTime.getTime()).toBeLessThanOrEqual(afterTest.getTime()); + }); +});