feat(protocol): add gRPC/Protobuf and JSON-RPC/MessagePack capture
Add capture and schema inference for gRPC/Protocol Buffers and JSON-RPC/MessagePack protocols (#364). - classifyProtocolFamily: detects protocol from content type or path - decodeJsonRpc: decodes JSON-RPC messages with credential redaction - decodeMessagePack: decodes MessagePack wire format (map/array detection) - decodeGrpcFrame: decodes gRPC 5-byte frame headers - inferMessageSchema: infers schema hypotheses from repeated messages - Credential-like fields are detected and redacted by default Closes #364
This commit is contained in:
morluto
parent
645668b70d
commit
3f336fefb1
|
|
@ -0,0 +1,272 @@
|
|||
import { z } from "zod";
|
||||
|
||||
import { jsonValueSchema } from "./jsonValue.js";
|
||||
|
||||
/** Supported protocol families for structured protocol capture. */
|
||||
export const protocolFamilySchema = z.enum([
|
||||
"grpc",
|
||||
"protobuf",
|
||||
"json-rpc",
|
||||
"messagepack",
|
||||
]);
|
||||
export type ProtocolFamily = z.infer<typeof protocolFamilySchema>;
|
||||
|
||||
/** A decoded field from a protocol message. */
|
||||
export const decodedFieldSchema = z.strictObject({
|
||||
/** Field name or path within the message structure. */
|
||||
path: z.string().min(1),
|
||||
/** Wire type or encoding hint. */
|
||||
wire_type: z.string().nullable(),
|
||||
/** Decoded field value. */
|
||||
value: jsonValueSchema.nullable(),
|
||||
/** Whether this field value was inferred rather than decoded. */
|
||||
inferred: z.boolean().default(false),
|
||||
});
|
||||
export type DecodedField = z.infer<typeof decodedFieldSchema>;
|
||||
|
||||
/** Schema inference hypothesis for a repeated message pattern. */
|
||||
export const schemaHypothesisSchema = z.strictObject({
|
||||
/** Hypothesized field name. */
|
||||
field_name: z.string().min(1),
|
||||
/** Inferred type. */
|
||||
inferred_type: z.string().min(1),
|
||||
/** Confidence: 0-1 */
|
||||
confidence: z.number().min(0).max(1),
|
||||
/** Supporting evidence count. */
|
||||
evidence_count: z.number().int().nonnegative(),
|
||||
});
|
||||
export type SchemaHypothesis = z.infer<typeof schemaHypothesisSchema>;
|
||||
|
||||
/** Result of decoding a single protocol message. */
|
||||
export const protocolMessageSchema = z.strictObject({
|
||||
/** Monotonic sequence number. */
|
||||
sequence: z.number().int().nonnegative(),
|
||||
/** Timestamp in milliseconds. */
|
||||
at_ms: z.number().int().nonnegative(),
|
||||
/** Protocol family. */
|
||||
family: protocolFamilySchema,
|
||||
/** Direction of the message. */
|
||||
direction: z.enum(["request", "response"]),
|
||||
/** Transport endpoint or path (e.g. gRPC method path). */
|
||||
endpoint: z.string().nullable(),
|
||||
/** Content type of the raw payload. */
|
||||
content_type: z.string().nullable(),
|
||||
/** Raw payload bytes as base64. */
|
||||
raw_payload: z.string().nullable(),
|
||||
/** Decoded fields from the payload. */
|
||||
decoded_fields: z.array(decodedFieldSchema).default([]),
|
||||
/** Inferred schema hypotheses. */
|
||||
schema_hypotheses: z.array(schemaHypothesisSchema).default([]),
|
||||
/** Whether decoding was truncated. */
|
||||
truncated: z.boolean().default(false),
|
||||
/** Whether credentials were detected and redacted. */
|
||||
credentials_redacted: z.boolean().default(false),
|
||||
});
|
||||
export type ProtocolMessage = z.infer<typeof protocolMessageSchema>;
|
||||
|
||||
/** A captured protocol session with messages and inferred schemas. */
|
||||
export const protocolCaptureSchema = z.strictObject({
|
||||
/** Protocol family for this capture. */
|
||||
family: protocolFamilySchema,
|
||||
/** Captured messages in order. */
|
||||
messages: z.array(protocolMessageSchema).min(0).max(10_000),
|
||||
/** Aggregated schema hypotheses across all messages. */
|
||||
inferred_schema: z.array(schemaHypothesisSchema).default([]),
|
||||
/** Whether any message was truncated. */
|
||||
has_truncated: z.boolean().default(false),
|
||||
/** Whether credentials were detected and redacted. */
|
||||
credentials_detected: z.boolean().default(false),
|
||||
});
|
||||
export type ProtocolCapture = z.infer<typeof protocolCaptureSchema>;
|
||||
|
||||
/** Classification of a byte sequence as a known protocol family. */
|
||||
export function classifyProtocolFamily(
|
||||
contentType: string | null,
|
||||
path: string | null,
|
||||
): ProtocolFamily | null {
|
||||
if (contentType) {
|
||||
if (contentType.includes("application/grpc")) return "grpc";
|
||||
if (contentType.includes("application/x-protobuf")) return "protobuf";
|
||||
if (contentType.includes("application/json-rpc")) return "json-rpc";
|
||||
if (
|
||||
contentType.includes("application/msgpack") ||
|
||||
contentType.includes("application/x-msgpack")
|
||||
)
|
||||
return "messagepack";
|
||||
}
|
||||
if (path) {
|
||||
if (path.startsWith("/grpc.") || path.includes("/grpc/")) return "grpc";
|
||||
if (path.includes("/json-rpc") || path.includes("/jsonrpc"))
|
||||
return "json-rpc";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Credential-like field names that should be redacted. */
|
||||
const CREDENTIAL_PATTERNS = [
|
||||
"password",
|
||||
"passwd",
|
||||
"secret",
|
||||
"token",
|
||||
"api_key",
|
||||
"apikey",
|
||||
"authorization",
|
||||
"auth",
|
||||
"credential",
|
||||
"private_key",
|
||||
];
|
||||
|
||||
/** Check if a field path looks like it may contain credentials. */
|
||||
export function looksLikeCredential(path: string): boolean {
|
||||
const lower = path.toLowerCase();
|
||||
return CREDENTIAL_PATTERNS.some((p) => lower.includes(p));
|
||||
}
|
||||
|
||||
/** Decode a JSON-RPC message from raw payload bytes. */
|
||||
export function decodeJsonRpc(rawPayload: Uint8Array): {
|
||||
decoded_fields: DecodedField[];
|
||||
truncated: boolean;
|
||||
} {
|
||||
const text = new TextDecoder().decode(rawPayload);
|
||||
const parsed = JSON.parse(text);
|
||||
const fields: DecodedField[] = [];
|
||||
|
||||
if (typeof parsed === "object" && parsed !== null) {
|
||||
for (const [key, value] of Object.entries(parsed)) {
|
||||
if (looksLikeCredential(key)) {
|
||||
fields.push({
|
||||
path: key,
|
||||
wire_type: "json",
|
||||
value: "[REDACTED]",
|
||||
inferred: false,
|
||||
});
|
||||
} else {
|
||||
fields.push({
|
||||
path: key,
|
||||
wire_type: "json",
|
||||
value: value as never,
|
||||
inferred: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { decoded_fields: fields, truncated: false };
|
||||
}
|
||||
|
||||
/** Decode a MessagePack message (simplified: only checks basic structure). */
|
||||
export function decodeMessagePack(rawPayload: Uint8Array): {
|
||||
decoded_fields: DecodedField[];
|
||||
truncated: boolean;
|
||||
} {
|
||||
const fields: DecodedField[] = [];
|
||||
// MessagePack uses a single-byte type tag
|
||||
if (rawPayload.length === 0) {
|
||||
return { decoded_fields: [], truncated: false };
|
||||
}
|
||||
const tag = rawPayload[0]!;
|
||||
// Map type: 0x80-0x8f (fixmap), 0xde (map16), 0xdf (map32)
|
||||
if ((tag >= 0x80 && tag <= 0x8f) || tag === 0xde || tag === 0xdf) {
|
||||
fields.push({
|
||||
path: "type",
|
||||
wire_type: "msgpack",
|
||||
value: "map",
|
||||
inferred: false,
|
||||
});
|
||||
}
|
||||
// Array type: 0x90-0x9f (fixarray), 0xdc (array16), 0xdd (array32)
|
||||
else if ((tag >= 0x90 && tag <= 0x9f) || tag === 0xdc || tag === 0xdd) {
|
||||
fields.push({
|
||||
path: "type",
|
||||
wire_type: "msgpack",
|
||||
value: "array",
|
||||
inferred: false,
|
||||
});
|
||||
} else {
|
||||
fields.push({
|
||||
path: "type",
|
||||
wire_type: "msgpack",
|
||||
value: "unknown",
|
||||
inferred: true,
|
||||
});
|
||||
}
|
||||
return {
|
||||
decoded_fields: fields,
|
||||
truncated: rawPayload.length > 10_000,
|
||||
};
|
||||
}
|
||||
|
||||
/** Decode a gRPC frame (5-byte header + protobuf payload). */
|
||||
export function decodeGrpcFrame(rawPayload: Uint8Array): {
|
||||
decoded_fields: DecodedField[];
|
||||
truncated: boolean;
|
||||
} {
|
||||
// gRPC uses a 5-byte prefix: 1 byte compressed flag + 4 byte length
|
||||
if (rawPayload.length < 5) {
|
||||
return {
|
||||
decoded_fields: [
|
||||
{
|
||||
path: "error",
|
||||
wire_type: "grpc",
|
||||
value: "frame too short",
|
||||
inferred: false,
|
||||
},
|
||||
],
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
const compressed = rawPayload[0] !== 0;
|
||||
const length =
|
||||
(rawPayload[1]! << 24) |
|
||||
(rawPayload[2]! << 16) |
|
||||
(rawPayload[3]! << 8) |
|
||||
rawPayload[4]!;
|
||||
return {
|
||||
decoded_fields: [
|
||||
{
|
||||
path: "compressed",
|
||||
wire_type: "grpc",
|
||||
value: compressed,
|
||||
inferred: false,
|
||||
},
|
||||
{
|
||||
path: "message_length",
|
||||
wire_type: "grpc",
|
||||
value: length,
|
||||
inferred: false,
|
||||
},
|
||||
],
|
||||
truncated: length > 10_000,
|
||||
};
|
||||
}
|
||||
|
||||
/** Infer schema hypotheses from a set of decoded messages. */
|
||||
export function inferMessageSchema(
|
||||
messages: readonly ProtocolMessage[],
|
||||
): SchemaHypothesis[] {
|
||||
const fieldCounts = new Map<string, { count: number; types: Set<string> }>();
|
||||
for (const msg of messages) {
|
||||
for (const field of msg.decoded_fields) {
|
||||
const existing = fieldCounts.get(field.path);
|
||||
if (existing) {
|
||||
existing.count++;
|
||||
if (field.wire_type) existing.types.add(field.wire_type);
|
||||
} else {
|
||||
fieldCounts.set(field.path, {
|
||||
count: 1,
|
||||
types: new Set(field.wire_type ?? "unknown"),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
const hypotheses: SchemaHypothesis[] = [];
|
||||
for (const [field_name, data] of fieldCounts) {
|
||||
hypotheses.push({
|
||||
field_name,
|
||||
inferred_type: [...data.types].join("|"),
|
||||
confidence: Math.min(1, data.count / messages.length || 0),
|
||||
evidence_count: data.count,
|
||||
});
|
||||
}
|
||||
return hypotheses;
|
||||
}
|
||||
|
|
@ -0,0 +1,207 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
classifyProtocolFamily,
|
||||
decodeGrpcFrame,
|
||||
decodeJsonRpc,
|
||||
decodeMessagePack,
|
||||
inferMessageSchema,
|
||||
looksLikeCredential,
|
||||
protocolCaptureSchema,
|
||||
type ProtocolMessage,
|
||||
type ProtocolCapture,
|
||||
} from "../src/domain/protocolCapture.js";
|
||||
|
||||
describe("protocol capture", () => {
|
||||
it("classifies gRPC from content type", () => {
|
||||
expect(classifyProtocolFamily("application/grpc+proto", null)).toBe("grpc");
|
||||
});
|
||||
|
||||
it("classifies protobuf from content type", () => {
|
||||
expect(classifyProtocolFamily("application/x-protobuf", null)).toBe(
|
||||
"protobuf",
|
||||
);
|
||||
});
|
||||
|
||||
it("classifies json-rpc from content type", () => {
|
||||
expect(classifyProtocolFamily("application/json-rpc", null)).toBe(
|
||||
"json-rpc",
|
||||
);
|
||||
});
|
||||
|
||||
it("classifies msgpack from content type", () => {
|
||||
expect(classifyProtocolFamily("application/msgpack", null)).toBe(
|
||||
"messagepack",
|
||||
);
|
||||
});
|
||||
|
||||
it("classifies from path when content type is null", () => {
|
||||
expect(classifyProtocolFamily(null, "/some/grpc/method")).toBe("grpc");
|
||||
expect(classifyProtocolFamily(null, "/api/json-rpc/2.0")).toBe("json-rpc");
|
||||
});
|
||||
|
||||
it("returns null for unknown protocols", () => {
|
||||
expect(classifyProtocolFamily("text/plain", null)).toBeNull();
|
||||
expect(classifyProtocolFamily(null, null)).toBeNull();
|
||||
});
|
||||
|
||||
it("detects credential-like field names", () => {
|
||||
expect(looksLikeCredential("password")).toBe(true);
|
||||
expect(looksLikeCredential("api_key")).toBe(true);
|
||||
expect(looksLikeCredential("authorization")).toBe(true);
|
||||
expect(looksLikeCredential("username")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("JSON-RPC decoding", () => {
|
||||
it("decodes a simple JSON-RPC request", () => {
|
||||
const payload = new TextEncoder().encode(
|
||||
JSON.stringify({
|
||||
jsonrpc: "2.0",
|
||||
method: "add",
|
||||
params: [1, 2],
|
||||
id: 1,
|
||||
}),
|
||||
);
|
||||
const result = decodeJsonRpc(payload);
|
||||
expect(result.decoded_fields.length).toBe(4);
|
||||
expect(result.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("redacts credential-like fields", () => {
|
||||
const payload = new TextEncoder().encode(
|
||||
JSON.stringify({
|
||||
password: "secret123",
|
||||
username: "admin",
|
||||
}),
|
||||
);
|
||||
const result = decodeJsonRpc(payload);
|
||||
const passwordField = result.decoded_fields.find(
|
||||
(f) => f.path === "password",
|
||||
);
|
||||
expect(passwordField?.value).toBe("[REDACTED]");
|
||||
const usernameField = result.decoded_fields.find(
|
||||
(f) => f.path === "username",
|
||||
);
|
||||
expect(usernameField?.value).toBe("admin");
|
||||
});
|
||||
});
|
||||
|
||||
describe("MessagePack decoding", () => {
|
||||
it("detects map type", () => {
|
||||
const payload = new Uint8Array([0x82]);
|
||||
const result = decodeMessagePack(payload);
|
||||
expect(result.decoded_fields[0]?.value).toBe("map");
|
||||
});
|
||||
|
||||
it("detects array type", () => {
|
||||
const payload = new Uint8Array([0x92]);
|
||||
const result = decodeMessagePack(payload);
|
||||
expect(result.decoded_fields[0]?.value).toBe("array");
|
||||
});
|
||||
|
||||
it("reports unknown type for other tags", () => {
|
||||
const payload = new Uint8Array([0xff]);
|
||||
const result = decodeMessagePack(payload);
|
||||
expect(result.decoded_fields[0]?.value).toBe("unknown");
|
||||
expect(result.decoded_fields[0]?.inferred).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("gRPC frame decoding", () => {
|
||||
it("decodes a valid gRPC frame header", () => {
|
||||
const payload = new Uint8Array([
|
||||
0x00, 0x00, 0x00, 0x00, 0x05, 1, 2, 3, 4, 5,
|
||||
]);
|
||||
const result = decodeGrpcFrame(payload);
|
||||
expect(result.decoded_fields).toHaveLength(2);
|
||||
expect(result.decoded_fields[0]?.path).toBe("compressed");
|
||||
expect(result.decoded_fields[0]?.value).toBe(false);
|
||||
expect(result.decoded_fields[1]?.path).toBe("message_length");
|
||||
expect(result.decoded_fields[1]?.value).toBe(5);
|
||||
});
|
||||
|
||||
it("detects compressed flag", () => {
|
||||
const payload = new Uint8Array([0x01, 0x00, 0x00, 0x00, 0x0a]);
|
||||
const result = decodeGrpcFrame(payload);
|
||||
expect(result.decoded_fields[0]?.value).toBe(true);
|
||||
});
|
||||
|
||||
it("handles too-short frames", () => {
|
||||
const payload = new Uint8Array([0x00, 0x01]);
|
||||
const result = decodeGrpcFrame(payload);
|
||||
expect(result.decoded_fields[0]?.path).toBe("error");
|
||||
});
|
||||
});
|
||||
|
||||
describe("schema inference", () => {
|
||||
it("infers schema from repeated messages", () => {
|
||||
const messages: ProtocolMessage[] = [
|
||||
{
|
||||
sequence: 0,
|
||||
at_ms: 0,
|
||||
family: "json-rpc",
|
||||
direction: "request",
|
||||
endpoint: "/api",
|
||||
content_type: "application/json",
|
||||
raw_payload: null,
|
||||
decoded_fields: [
|
||||
{ path: "method", wire_type: "json", value: "add", inferred: false },
|
||||
{ path: "id", wire_type: "json", value: 1, inferred: false },
|
||||
],
|
||||
schema_hypotheses: [],
|
||||
truncated: false,
|
||||
credentials_redacted: false,
|
||||
},
|
||||
{
|
||||
sequence: 1,
|
||||
at_ms: 10,
|
||||
family: "json-rpc",
|
||||
direction: "response",
|
||||
endpoint: "/api",
|
||||
content_type: "application/json",
|
||||
raw_payload: null,
|
||||
decoded_fields: [
|
||||
{ path: "result", wire_type: "json", value: 3, inferred: false },
|
||||
{ path: "id", wire_type: "json", value: 1, inferred: false },
|
||||
],
|
||||
schema_hypotheses: [],
|
||||
truncated: false,
|
||||
credentials_redacted: false,
|
||||
},
|
||||
];
|
||||
const hypotheses = inferMessageSchema(messages);
|
||||
expect(hypotheses).toHaveLength(3);
|
||||
const idHypothesis = hypotheses.find((h) => h.field_name === "id");
|
||||
expect(idHypothesis).toBeDefined();
|
||||
expect(idHypothesis!.evidence_count).toBe(2);
|
||||
expect(idHypothesis!.confidence).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("uses ProtocolCapture type", () => {
|
||||
it("validates a well-formed capture typed value", () => {
|
||||
const capture: ProtocolCapture = {
|
||||
family: "grpc",
|
||||
messages: [],
|
||||
inferred_schema: [],
|
||||
has_truncated: false,
|
||||
credentials_detected: false,
|
||||
};
|
||||
expect(capture.family).toBe("grpc");
|
||||
});
|
||||
});
|
||||
|
||||
describe("protocol capture schema validation", () => {
|
||||
it("validates a well-formed capture", () => {
|
||||
const capture = {
|
||||
family: "grpc" as const,
|
||||
messages: [],
|
||||
inferred_schema: [],
|
||||
has_truncated: false,
|
||||
credentials_detected: false,
|
||||
};
|
||||
const result = protocolCaptureSchema.safeParse(capture);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue