diff --git a/src/domain/protocolCapture.ts b/src/domain/protocolCapture.ts new file mode 100644 index 000000000..9c6dedd2c --- /dev/null +++ b/src/domain/protocolCapture.ts @@ -0,0 +1,272 @@ +import { z } from "zod"; + +import { jsonValueSchema } from "./jsonValue.js"; + +/** Supported protocol families for structured protocol capture. */ +export const protocolFamilySchema = z.enum([ + "grpc", + "protobuf", + "json-rpc", + "messagepack", +]); +export type ProtocolFamily = z.infer; + +/** A decoded field from a protocol message. */ +export const decodedFieldSchema = z.strictObject({ + /** Field name or path within the message structure. */ + path: z.string().min(1), + /** Wire type or encoding hint. */ + wire_type: z.string().nullable(), + /** Decoded field value. */ + value: jsonValueSchema.nullable(), + /** Whether this field value was inferred rather than decoded. */ + inferred: z.boolean().default(false), +}); +export type DecodedField = z.infer; + +/** Schema inference hypothesis for a repeated message pattern. */ +export const schemaHypothesisSchema = z.strictObject({ + /** Hypothesized field name. */ + field_name: z.string().min(1), + /** Inferred type. */ + inferred_type: z.string().min(1), + /** Confidence: 0-1 */ + confidence: z.number().min(0).max(1), + /** Supporting evidence count. */ + evidence_count: z.number().int().nonnegative(), +}); +export type SchemaHypothesis = z.infer; + +/** Result of decoding a single protocol message. */ +export const protocolMessageSchema = z.strictObject({ + /** Monotonic sequence number. */ + sequence: z.number().int().nonnegative(), + /** Timestamp in milliseconds. */ + at_ms: z.number().int().nonnegative(), + /** Protocol family. */ + family: protocolFamilySchema, + /** Direction of the message. */ + direction: z.enum(["request", "response"]), + /** Transport endpoint or path (e.g. gRPC method path). */ + endpoint: z.string().nullable(), + /** Content type of the raw payload. */ + content_type: z.string().nullable(), + /** Raw payload bytes as base64. */ + raw_payload: z.string().nullable(), + /** Decoded fields from the payload. */ + decoded_fields: z.array(decodedFieldSchema).default([]), + /** Inferred schema hypotheses. */ + schema_hypotheses: z.array(schemaHypothesisSchema).default([]), + /** Whether decoding was truncated. */ + truncated: z.boolean().default(false), + /** Whether credentials were detected and redacted. */ + credentials_redacted: z.boolean().default(false), +}); +export type ProtocolMessage = z.infer; + +/** A captured protocol session with messages and inferred schemas. */ +export const protocolCaptureSchema = z.strictObject({ + /** Protocol family for this capture. */ + family: protocolFamilySchema, + /** Captured messages in order. */ + messages: z.array(protocolMessageSchema).min(0).max(10_000), + /** Aggregated schema hypotheses across all messages. */ + inferred_schema: z.array(schemaHypothesisSchema).default([]), + /** Whether any message was truncated. */ + has_truncated: z.boolean().default(false), + /** Whether credentials were detected and redacted. */ + credentials_detected: z.boolean().default(false), +}); +export type ProtocolCapture = z.infer; + +/** Classification of a byte sequence as a known protocol family. */ +export function classifyProtocolFamily( + contentType: string | null, + path: string | null, +): ProtocolFamily | null { + if (contentType) { + if (contentType.includes("application/grpc")) return "grpc"; + if (contentType.includes("application/x-protobuf")) return "protobuf"; + if (contentType.includes("application/json-rpc")) return "json-rpc"; + if ( + contentType.includes("application/msgpack") || + contentType.includes("application/x-msgpack") + ) + return "messagepack"; + } + if (path) { + if (path.startsWith("/grpc.") || path.includes("/grpc/")) return "grpc"; + if (path.includes("/json-rpc") || path.includes("/jsonrpc")) + return "json-rpc"; + } + return null; +} + +/** Credential-like field names that should be redacted. */ +const CREDENTIAL_PATTERNS = [ + "password", + "passwd", + "secret", + "token", + "api_key", + "apikey", + "authorization", + "auth", + "credential", + "private_key", +]; + +/** Check if a field path looks like it may contain credentials. */ +export function looksLikeCredential(path: string): boolean { + const lower = path.toLowerCase(); + return CREDENTIAL_PATTERNS.some((p) => lower.includes(p)); +} + +/** Decode a JSON-RPC message from raw payload bytes. */ +export function decodeJsonRpc(rawPayload: Uint8Array): { + decoded_fields: DecodedField[]; + truncated: boolean; +} { + const text = new TextDecoder().decode(rawPayload); + const parsed = JSON.parse(text); + const fields: DecodedField[] = []; + + if (typeof parsed === "object" && parsed !== null) { + for (const [key, value] of Object.entries(parsed)) { + if (looksLikeCredential(key)) { + fields.push({ + path: key, + wire_type: "json", + value: "[REDACTED]", + inferred: false, + }); + } else { + fields.push({ + path: key, + wire_type: "json", + value: value as never, + inferred: false, + }); + } + } + } + + return { decoded_fields: fields, truncated: false }; +} + +/** Decode a MessagePack message (simplified: only checks basic structure). */ +export function decodeMessagePack(rawPayload: Uint8Array): { + decoded_fields: DecodedField[]; + truncated: boolean; +} { + const fields: DecodedField[] = []; + // MessagePack uses a single-byte type tag + if (rawPayload.length === 0) { + return { decoded_fields: [], truncated: false }; + } + const tag = rawPayload[0]!; + // Map type: 0x80-0x8f (fixmap), 0xde (map16), 0xdf (map32) + if ((tag >= 0x80 && tag <= 0x8f) || tag === 0xde || tag === 0xdf) { + fields.push({ + path: "type", + wire_type: "msgpack", + value: "map", + inferred: false, + }); + } + // Array type: 0x90-0x9f (fixarray), 0xdc (array16), 0xdd (array32) + else if ((tag >= 0x90 && tag <= 0x9f) || tag === 0xdc || tag === 0xdd) { + fields.push({ + path: "type", + wire_type: "msgpack", + value: "array", + inferred: false, + }); + } else { + fields.push({ + path: "type", + wire_type: "msgpack", + value: "unknown", + inferred: true, + }); + } + return { + decoded_fields: fields, + truncated: rawPayload.length > 10_000, + }; +} + +/** Decode a gRPC frame (5-byte header + protobuf payload). */ +export function decodeGrpcFrame(rawPayload: Uint8Array): { + decoded_fields: DecodedField[]; + truncated: boolean; +} { + // gRPC uses a 5-byte prefix: 1 byte compressed flag + 4 byte length + if (rawPayload.length < 5) { + return { + decoded_fields: [ + { + path: "error", + wire_type: "grpc", + value: "frame too short", + inferred: false, + }, + ], + truncated: false, + }; + } + const compressed = rawPayload[0] !== 0; + const length = + (rawPayload[1]! << 24) | + (rawPayload[2]! << 16) | + (rawPayload[3]! << 8) | + rawPayload[4]!; + return { + decoded_fields: [ + { + path: "compressed", + wire_type: "grpc", + value: compressed, + inferred: false, + }, + { + path: "message_length", + wire_type: "grpc", + value: length, + inferred: false, + }, + ], + truncated: length > 10_000, + }; +} + +/** Infer schema hypotheses from a set of decoded messages. */ +export function inferMessageSchema( + messages: readonly ProtocolMessage[], +): SchemaHypothesis[] { + const fieldCounts = new Map }>(); + for (const msg of messages) { + for (const field of msg.decoded_fields) { + const existing = fieldCounts.get(field.path); + if (existing) { + existing.count++; + if (field.wire_type) existing.types.add(field.wire_type); + } else { + fieldCounts.set(field.path, { + count: 1, + types: new Set(field.wire_type ?? "unknown"), + }); + } + } + } + const hypotheses: SchemaHypothesis[] = []; + for (const [field_name, data] of fieldCounts) { + hypotheses.push({ + field_name, + inferred_type: [...data.types].join("|"), + confidence: Math.min(1, data.count / messages.length || 0), + evidence_count: data.count, + }); + } + return hypotheses; +} diff --git a/tests/protocolCapture.test.ts b/tests/protocolCapture.test.ts new file mode 100644 index 000000000..23dac0e3a --- /dev/null +++ b/tests/protocolCapture.test.ts @@ -0,0 +1,207 @@ +import { describe, expect, it } from "vitest"; + +import { + classifyProtocolFamily, + decodeGrpcFrame, + decodeJsonRpc, + decodeMessagePack, + inferMessageSchema, + looksLikeCredential, + protocolCaptureSchema, + type ProtocolMessage, + type ProtocolCapture, +} from "../src/domain/protocolCapture.js"; + +describe("protocol capture", () => { + it("classifies gRPC from content type", () => { + expect(classifyProtocolFamily("application/grpc+proto", null)).toBe("grpc"); + }); + + it("classifies protobuf from content type", () => { + expect(classifyProtocolFamily("application/x-protobuf", null)).toBe( + "protobuf", + ); + }); + + it("classifies json-rpc from content type", () => { + expect(classifyProtocolFamily("application/json-rpc", null)).toBe( + "json-rpc", + ); + }); + + it("classifies msgpack from content type", () => { + expect(classifyProtocolFamily("application/msgpack", null)).toBe( + "messagepack", + ); + }); + + it("classifies from path when content type is null", () => { + expect(classifyProtocolFamily(null, "/some/grpc/method")).toBe("grpc"); + expect(classifyProtocolFamily(null, "/api/json-rpc/2.0")).toBe("json-rpc"); + }); + + it("returns null for unknown protocols", () => { + expect(classifyProtocolFamily("text/plain", null)).toBeNull(); + expect(classifyProtocolFamily(null, null)).toBeNull(); + }); + + it("detects credential-like field names", () => { + expect(looksLikeCredential("password")).toBe(true); + expect(looksLikeCredential("api_key")).toBe(true); + expect(looksLikeCredential("authorization")).toBe(true); + expect(looksLikeCredential("username")).toBe(false); + }); +}); + +describe("JSON-RPC decoding", () => { + it("decodes a simple JSON-RPC request", () => { + const payload = new TextEncoder().encode( + JSON.stringify({ + jsonrpc: "2.0", + method: "add", + params: [1, 2], + id: 1, + }), + ); + const result = decodeJsonRpc(payload); + expect(result.decoded_fields.length).toBe(4); + expect(result.truncated).toBe(false); + }); + + it("redacts credential-like fields", () => { + const payload = new TextEncoder().encode( + JSON.stringify({ + password: "secret123", + username: "admin", + }), + ); + const result = decodeJsonRpc(payload); + const passwordField = result.decoded_fields.find( + (f) => f.path === "password", + ); + expect(passwordField?.value).toBe("[REDACTED]"); + const usernameField = result.decoded_fields.find( + (f) => f.path === "username", + ); + expect(usernameField?.value).toBe("admin"); + }); +}); + +describe("MessagePack decoding", () => { + it("detects map type", () => { + const payload = new Uint8Array([0x82]); + const result = decodeMessagePack(payload); + expect(result.decoded_fields[0]?.value).toBe("map"); + }); + + it("detects array type", () => { + const payload = new Uint8Array([0x92]); + const result = decodeMessagePack(payload); + expect(result.decoded_fields[0]?.value).toBe("array"); + }); + + it("reports unknown type for other tags", () => { + const payload = new Uint8Array([0xff]); + const result = decodeMessagePack(payload); + expect(result.decoded_fields[0]?.value).toBe("unknown"); + expect(result.decoded_fields[0]?.inferred).toBe(true); + }); +}); + +describe("gRPC frame decoding", () => { + it("decodes a valid gRPC frame header", () => { + const payload = new Uint8Array([ + 0x00, 0x00, 0x00, 0x00, 0x05, 1, 2, 3, 4, 5, + ]); + const result = decodeGrpcFrame(payload); + expect(result.decoded_fields).toHaveLength(2); + expect(result.decoded_fields[0]?.path).toBe("compressed"); + expect(result.decoded_fields[0]?.value).toBe(false); + expect(result.decoded_fields[1]?.path).toBe("message_length"); + expect(result.decoded_fields[1]?.value).toBe(5); + }); + + it("detects compressed flag", () => { + const payload = new Uint8Array([0x01, 0x00, 0x00, 0x00, 0x0a]); + const result = decodeGrpcFrame(payload); + expect(result.decoded_fields[0]?.value).toBe(true); + }); + + it("handles too-short frames", () => { + const payload = new Uint8Array([0x00, 0x01]); + const result = decodeGrpcFrame(payload); + expect(result.decoded_fields[0]?.path).toBe("error"); + }); +}); + +describe("schema inference", () => { + it("infers schema from repeated messages", () => { + const messages: ProtocolMessage[] = [ + { + sequence: 0, + at_ms: 0, + family: "json-rpc", + direction: "request", + endpoint: "/api", + content_type: "application/json", + raw_payload: null, + decoded_fields: [ + { path: "method", wire_type: "json", value: "add", inferred: false }, + { path: "id", wire_type: "json", value: 1, inferred: false }, + ], + schema_hypotheses: [], + truncated: false, + credentials_redacted: false, + }, + { + sequence: 1, + at_ms: 10, + family: "json-rpc", + direction: "response", + endpoint: "/api", + content_type: "application/json", + raw_payload: null, + decoded_fields: [ + { path: "result", wire_type: "json", value: 3, inferred: false }, + { path: "id", wire_type: "json", value: 1, inferred: false }, + ], + schema_hypotheses: [], + truncated: false, + credentials_redacted: false, + }, + ]; + const hypotheses = inferMessageSchema(messages); + expect(hypotheses).toHaveLength(3); + const idHypothesis = hypotheses.find((h) => h.field_name === "id"); + expect(idHypothesis).toBeDefined(); + expect(idHypothesis!.evidence_count).toBe(2); + expect(idHypothesis!.confidence).toBe(1); + }); +}); + +describe("uses ProtocolCapture type", () => { + it("validates a well-formed capture typed value", () => { + const capture: ProtocolCapture = { + family: "grpc", + messages: [], + inferred_schema: [], + has_truncated: false, + credentials_detected: false, + }; + expect(capture.family).toBe("grpc"); + }); +}); + +describe("protocol capture schema validation", () => { + it("validates a well-formed capture", () => { + const capture = { + family: "grpc" as const, + messages: [], + inferred_schema: [], + has_truncated: false, + credentials_detected: false, + }; + const result = protocolCaptureSchema.safeParse(capture); + expect(result.success).toBe(true); + }); +});