store statsig

This commit is contained in:
Hanashi 2026-08-24 01:39:24 -04:00
parent 6831df7c63
commit 00a25ea609
14 changed files with 268 additions and 75 deletions

1
Cargo.lock generated
View File

@ -2634,6 +2634,7 @@ dependencies = [
"log",
"serde",
"serde_json",
"sha2 0.11.0-rc.5",
"tao",
"thiserror 2.0.18",
"tokio",

View File

@ -30,6 +30,7 @@ chrono.workspace = true
log.workspace = true
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
sha2.workspace = true
thiserror.workspace = true
tokio = { version = "1", features = ["sync", "time", "macros", "rt"] }
toml.workspace = true

View File

@ -56,6 +56,8 @@
mod browser_context;
mod mint_harness;
mod minted_statsig;
mod statsig;
mod statsig_material;
mod statsig_minter;
mod statsig_oracle;
mod statsig_request;
@ -68,6 +70,8 @@ pub mod harvest_via_webview;
pub use browser_context::{BrowserContext, DEFAULT_USER_AGENT};
pub use mint_harness::MINT_HARNESS_SCRIPT;
pub use minted_statsig::MintedStatsig;
pub use statsig::Statsig;
pub use statsig_material::StatsigMaterial;
pub use statsig_minter::{MinterConfig, StatsigMinter};
pub use statsig_oracle::{StatsigError, StatsigOracle};
pub use statsig_request::StatsigRequest;

View File

@ -0,0 +1,22 @@
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
/// A concrete, ready-to-send `x-statsig-id`, minted for one endpoint at one
/// time from [`StatsigMaterial`](crate::StatsigMaterial).
///
/// The embedded timestamp ages out, so a `Statsig` is short-lived — regenerate
/// per request rather than caching it for long.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Statsig {
/// The header value to send as `x-statsig-id`.
pub statsig: String,
/// When it was generated (this is the timestamp baked into the signature).
pub generated_at: DateTime<Utc>,
/// The HTTP method the signature is bound to (uppercase).
pub method: String,
/// The request path the signature is bound to.
pub path: String,
}

View File

@ -0,0 +1,149 @@
//! [`StatsigMaterial`] — the reusable, per-session material an
//! `x-statsig-id` is minted from, plus the pure signature assembly.
use crate::statsig::Statsig;
use base64::prelude::BASE64_STANDARD_NO_PAD;
use base64::Engine;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
/// Unix-time reference the embedded `number` counts from (2023-05-01).
const STATSIG_EPOCH: i64 = 1_682_924_400;
/// Salt folded into the signed message (best-known; absent from current bundles).
const STATSIG_SALT: &str = "obfiowerehiring";
/// Trailing marker byte of the assembled payload.
const STATSIG_MARK: u8 = 0x03;
/// The reusable material an `x-statsig-id` is minted from, independent of the
/// endpoint. A signature binds `method`, `path`, and a timestamp — all
/// per-request — around a "genuine hex" fingerprint **derived from the seed**.
/// So the one thing worth storing across requests is the 48-byte **seed**
/// (decoded from a real captured signature): with it we can mint a fresh
/// signature for any endpoint via [`generate_statsig`](Self::generate_statsig),
/// no browser required — see the caveat there.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct StatsigMaterial {
/// The 48-byte seed, base64 (standard alphabet, no padding).
pub seed_b64: String,
}
impl StatsigMaterial {
pub fn from_seed_b64(seed_b64: impl Into<String>) -> Self {
Self { seed_b64: seed_b64.into() }
}
/// Mint a signature for `method` + `path` stamped at `generated_at`.
///
/// # Caveat: needs the seed → hex fingerprint
///
/// The signature folds in a "genuine hex" that Grok derives from the seed by
/// sampling a CSS-animated SVG in-browser, rotating the constants over time.
/// [`compute_genuine_hex`] — the headless port of that — is currently
/// unavailable (stale + out of scope to extract), so this returns `None`.
/// The *assembly* around it ([`assemble`]) is complete and tested, so the
/// moment a real fingerprint lands, on-the-fly generation works.
pub fn generate_statsig(&self, method: &str, path: &str, generated_at: DateTime<Utc>) -> Option<Statsig> {
let seed = BASE64_STANDARD_NO_PAD.decode(self.seed_b64.trim()).ok()?;
let genuine_hex = compute_genuine_hex(&seed)?;
Some(assemble(&seed, method, path, generated_at, &genuine_hex))
}
}
/// Assemble the 70-byte payload from every input (pure, deterministic).
///
/// Layout: `[key][seed^key (48)][number_LE^key (4)][sha256(msg)[..16]^key (16)][mark^key]`,
/// where `msg = METHOD!path!<number-decimal><salt><genuine_hex>` and
/// `number = generated_at - epoch`.
fn assemble(seed: &[u8], method: &str, path: &str, generated_at: DateTime<Utc>, genuine_hex: &str) -> Statsig {
let method = method.trim().to_ascii_uppercase();
let path = path.trim();
let number = generated_at.timestamp().saturating_sub(STATSIG_EPOCH).max(0) as u32;
let message = format!("{method}!{path}!{number}{STATSIG_SALT}{genuine_hex}");
let digest = Sha256::digest(message.as_bytes());
// Any byte works as the XOR mask; derive one deterministically from the seed
// and timestamp so output is reproducible.
let key = seed.first().copied().unwrap_or(0) ^ (number as u8);
let number_bytes = number.to_le_bytes();
let mut out = Vec::with_capacity(70);
out.push(key);
out.extend(seed.iter().map(|byte| byte ^ key));
out.extend(number_bytes.iter().map(|byte| byte ^ key));
out.extend(digest[..16].iter().map(|byte| byte ^ key));
out.push(STATSIG_MARK ^ key);
Statsig {
statsig: BASE64_STANDARD_NO_PAD.encode(out),
generated_at,
method,
path: path.to_string(),
}
}
/// Grok's seed → "genuine hex" fingerprint. Currently unavailable: Grok derives
/// it in-browser by sampling a rotating CSS-animated SVG, and the headless port
/// is stale (extracting the live one is out of scope / blocked). Until this
/// returns a real value, [`StatsigMaterial::generate_statsig`] yields `None`.
/// This is the single missing piece — the assembly around it is done.
fn compute_genuine_hex(_seed: &[u8]) -> Option<String> {
None
}
#[cfg(test)]
mod tests {
use super::*;
const SEED: [u8; 48] = [7u8; 48];
const NEW_CONVERSATION: &str = "/rest/app-chat/conversations/new";
#[test]
fn assemble_produces_a_self_consistent_signature() {
let generated_at = DateTime::from_timestamp(1_787_535_100, 0).unwrap();
let hex = "3bab9506b851eb85"; // any fingerprint; the assembly is what's under test
let signature = assemble(&SEED, "post", NEW_CONVERSATION, generated_at, hex);
assert_eq!(signature.method, "POST");
assert_eq!(signature.path, NEW_CONVERSATION);
// Decode the payload and check every region reconstructs its input.
let raw = BASE64_STANDARD_NO_PAD.decode(&signature.statsig).unwrap();
assert_eq!(raw.len(), 70);
let key = raw[0];
let unmask = |slice: &[u8]| -> Vec<u8> { slice.iter().map(|b| b ^ key).collect() };
assert_eq!(unmask(&raw[1..49]), SEED);
let number_bytes = unmask(&raw[49..53]);
let number = u32::from_le_bytes([number_bytes[0], number_bytes[1], number_bytes[2], number_bytes[3]]);
assert_eq!(number as i64, 1_787_535_100 - STATSIG_EPOCH);
let digest = unmask(&raw[53..69]);
let expected = &Sha256::digest(
format!("POST!{NEW_CONVERSATION}!{number}{STATSIG_SALT}{hex}").as_bytes(),
)[..16];
assert_eq!(digest, expected);
assert_eq!(raw[69] ^ key, STATSIG_MARK);
}
#[test]
fn same_seed_different_endpoints_differ() {
let at = DateTime::from_timestamp(1_787_535_100, 0).unwrap();
let a = assemble(&SEED, "POST", "/a", at, "beef");
let b = assemble(&SEED, "POST", "/b", at, "beef");
assert_ne!(a.statsig, b.statsig, "different paths must sign differently");
}
#[test]
fn generate_statsig_is_none_until_fingerprint_available() {
// The seed is present, but the seed -> hex fingerprint is not, so no
// signature can be minted yet.
let material = StatsigMaterial::from_seed_b64(BASE64_STANDARD_NO_PAD.encode(SEED));
assert!(material.generate_statsig("POST", NEW_CONVERSATION, Utc::now()).is_none());
}
}

View File

@ -6,10 +6,11 @@ use crate::credentials::credential_user_info::CredentialUserInfo;
use crate::credentials::service_cookie_origin::cookie_origin_for_service;
use crate::error::artcraftx_error::ArtcraftXError;
use cookie_store_wrapper::cookie_store::CookieStore;
use crate::credentials::cookie_credential_grok_extra_pieces::CookieCredentialGrokExtraPieces;
use crate::state::data_dir::app_data_root::AppDataRoot;
use crate::state::data_dir::subdirectory::app_credentials_dir::AppCredentialsDir;
use chrono::{Duration, Utc};
use grok_consumer_statsig::statsig_cache_file::CapturedStatsig;
use chrono::Utc;
use grok_consumer_statsig::StatsigMaterial;
use log::{error, info};
use serde_derive::Serialize;
use tauri::State;
@ -23,8 +24,8 @@ pub struct WebCredentialSave {
pub service: GenerationSource,
pub cookies: CookieStore,
pub maybe_user_info: Option<CredentialUserInfo>,
/// Preemptively captured statsig pieces (Grok only; `None` for other sites).
pub maybe_statsig: Option<CapturedStatsig>,
/// Preemptively captured statsig material (Grok only; `None` for other sites).
pub maybe_statsig: Option<StatsigMaterial>,
}
/// Save a web-login (cookie) credential to the credentials directory.
@ -47,28 +48,21 @@ pub fn save_web_credential(
let now = Utc::now();
// A fresh capture stamps fetched/refresh times; without one, preserve
// whatever the existing credential already had.
let (statsig, statsig_fetched_at, statsig_refresh_at) = match save.maybe_statsig {
Some(captured) => (
Some(captured),
Some(now),
Some(now + Duration::minutes(STATSIG_REFRESH_MINUTES)),
),
// Fresh material stamps fetched/refresh times; without one, preserve whatever
// the existing credential already had.
let grok_data = match save.maybe_statsig {
Some(material) => Some(CookieCredentialGrokExtraPieces::fresh(material, now, STATSIG_REFRESH_MINUTES)),
None => existing
.as_ref()
.and_then(|credential| credential.cookies())
.map(|cookie| (cookie.statsig.clone(), cookie.statsig_fetched_at, cookie.statsig_refresh_at))
.unwrap_or((None, None, None)),
.and_then(|cookie| cookie.grok_data.clone()),
};
let cookie = CookieCredential {
updated_at: Some(now),
failed_at: None,
succeeded_at: None,
statsig_fetched_at,
statsig_refresh_at,
statsig,
grok_data,
cookies: save.cookies,
};

View File

@ -140,9 +140,7 @@ fn save_session_credential(
updated_at: Some(now),
failed_at: None,
succeeded_at: Some(now),
statsig_fetched_at: None,
statsig_refresh_at: None,
statsig: None,
grok_data: None,
cookies: CookieStore::from_cookie_header(
&format!("session={}", signed_session), &cookie_origin),
};

View File

@ -51,9 +51,7 @@ pub async fn edit_web_credential_command(
let existing_cookies = existing_cookie.cookies.clone();
let existing_failed_at = existing_cookie.failed_at;
let existing_succeeded_at = existing_cookie.succeeded_at;
let existing_statsig = existing_cookie.statsig.clone();
let existing_statsig_fetched_at = existing_cookie.statsig_fetched_at;
let existing_statsig_refresh_at = existing_cookie.statsig_refresh_at;
let existing_grok_data = existing_cookie.grok_data.clone();
if let Some(new_cookie_header) = cookie_header {
let new_cookie_header = new_cookie_header.trim().to_string();
@ -69,9 +67,7 @@ pub async fn edit_web_credential_command(
updated_at: Some(Utc::now()),
failed_at: existing_failed_at,
succeeded_at: existing_succeeded_at,
statsig_fetched_at: existing_statsig_fetched_at,
statsig_refresh_at: existing_statsig_refresh_at,
statsig: existing_statsig,
grok_data: existing_grok_data,
cookies: new_cookies,
});
}

View File

@ -99,19 +99,19 @@ impl AuthCredential {
mod tests {
use super::*;
// A cookie credential carrying captured statsig pieces must round-trip: the
// statsig table has to serialize before the `cookies` array-of-tables, or the
// TOML is invalid.
// A cookie credential carrying Grok statsig data must round-trip: the
// `grok_data` table has to serialize before the `cookies` array-of-tables, or
// the TOML is invalid.
#[test]
fn cookie_credential_with_statsig_round_trips() {
fn cookie_credential_with_grok_data_round_trips() {
use crate::credentials::cookie_credential::CookieCredential;
use chrono::{Duration, Utc};
use crate::credentials::cookie_credential_grok_extra_pieces::CookieCredentialGrokExtraPieces;
use chrono::Utc;
use cookie_store_wrapper::cookie_store::CookieStore;
use grok_consumer_statsig::statsig_cache_file::decode_statsig;
use grok_consumer_statsig::StatsigMaterial;
use reqwest::Url;
let capture = "XbHvyYh7hmNc+8sVAGnrpcuHN/MK+mgN63cmnBljOJaoCKu6zTPVw6C7HVv1wculKCNmYVum+L1h0IjFmtyZD/C53M+ZXg";
let statsig = decode_statsig(capture, "POST", "/rest/app-chat/conversations/new", 1_787_535_100).unwrap();
let material = StatsigMaterial::from_seed_b64("7LKU1SbbPgGmlkhdNLb4ltpqrlenNVC2KnvBRD5ly/VV9ueQboie/eZABqiclvh1");
let mut cookies = CookieStore::empty();
cookies.insert_named("sso", "token-value", &Url::parse("https://grok.com/").unwrap());
@ -119,9 +119,7 @@ mod tests {
let now = Utc::now();
let mut cookie = CookieCredential::new(cookies);
cookie.updated_at = Some(now);
cookie.statsig_fetched_at = Some(now);
cookie.statsig_refresh_at = Some(now + Duration::minutes(30));
cookie.statsig = Some(statsig.clone());
cookie.grok_data = Some(CookieCredentialGrokExtraPieces::fresh(material.clone(), now, 30));
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("xai_cookies.toml");
@ -136,11 +134,11 @@ mod tests {
credential.save().unwrap();
let loaded = AuthCredential::load_from_file(&path).unwrap();
let loaded_cookie = loaded.cookies().unwrap();
assert_eq!(loaded_cookie.statsig.as_ref(), Some(&statsig));
assert!(loaded_cookie.statsig_fetched_at.is_some());
assert!(loaded_cookie.statsig_refresh_at.is_some());
assert!(loaded_cookie.cookies.has_cookie("sso"));
let loaded_grok = loaded.cookies().unwrap().grok_data.as_ref().unwrap();
assert_eq!(loaded_grok.statsig_material.as_ref(), Some(&material));
assert!(loaded_grok.statsig_fetched_at.is_some());
assert!(loaded_grok.statsig_refresh_at.is_some());
assert!(loaded.cookies().unwrap().cookies.has_cookie("sso"));
}
#[test]

View File

@ -1,6 +1,6 @@
use crate::credentials::cookie_credential_grok_extra_pieces::CookieCredentialGrokExtraPieces;
use chrono::{DateTime, Utc};
use cookie_store_wrapper::cookie_store::CookieStore;
use grok_consumer_statsig::statsig_cache_file::CapturedStatsig;
use serde_derive::{Deserialize, Serialize};
/// A browser-cookie secret for a website integration.
@ -18,19 +18,10 @@ pub struct CookieCredential {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub succeeded_at: Option<DateTime<Utc>>,
/// When the stored [`statsig`](Self::statsig) pieces were captured. Grok only.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub statsig_fetched_at: Option<DateTime<Utc>>,
/// When the statsig pieces should be re-captured (30 min after fetch). Grok only.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub statsig_refresh_at: Option<DateTime<Utc>>,
/// Preemptively captured `x-statsig-id` pieces (Grok only). Decoded from a
/// real browser request during login via `grok_consumer_statsig`.
/// Grok-only extras: the reusable statsig material and its refresh schedule.
/// NB: a table, so it must stay after all scalar fields (before `cookies`).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub statsig: Option<CapturedStatsig>,
pub grok_data: Option<CookieCredentialGrokExtraPieces>,
/// The stored cookies with their RFC 6265 attributes.
/// NB: Serialized as an array of tables, so this field must stay LAST for
@ -44,9 +35,7 @@ impl CookieCredential {
updated_at: None,
failed_at: None,
succeeded_at: None,
statsig_fetched_at: None,
statsig_refresh_at: None,
statsig: None,
grok_data: None,
cookies,
}
}

View File

@ -0,0 +1,37 @@
use chrono::{DateTime, Utc};
use grok_consumer_statsig::StatsigMaterial;
use serde_derive::{Deserialize, Serialize};
/// Grok-specific extras stored alongside the cookies in a
/// [`CookieCredential`](crate::credentials::cookie_credential::CookieCredential).
///
/// Holds the reusable statsig material (the seed a fresh `x-statsig-id` is
/// minted from for any endpoint — see [`StatsigMaterial`]) plus when it was
/// captured and when it should be re-captured.
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
pub struct CookieCredentialGrokExtraPieces {
/// When the statsig material was captured.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub statsig_fetched_at: Option<DateTime<Utc>>,
/// When the statsig material should be re-captured (30 min after fetch).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub statsig_refresh_at: Option<DateTime<Utc>>,
/// The reusable statsig material (currently just the captured seed).
/// NB: a table, so it must stay after the scalar timestamp fields.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub statsig_material: Option<StatsigMaterial>,
}
impl CookieCredentialGrokExtraPieces {
/// Stamp fresh material with a fetch time and a refresh time
/// `refresh_minutes` into the future.
pub fn fresh(material: StatsigMaterial, now: DateTime<Utc>, refresh_minutes: i64) -> Self {
Self {
statsig_fetched_at: Some(now),
statsig_refresh_at: Some(now + chrono::Duration::minutes(refresh_minutes)),
statsig_material: Some(material),
}
}
}

View File

@ -18,6 +18,7 @@
pub mod api_key_credential;
pub mod auth_credential;
pub mod cookie_credential;
pub mod cookie_credential_grok_extra_pieces;
pub mod credential_toml;
pub mod credential_user_info;
pub mod find_service_credentials;

View File

@ -135,11 +135,11 @@ fn check_login_window(
info!("Detected {} login identity: {:?}", website, user_info);
}
// Preemptively grab the statsig pieces the capture harness stashed (Grok
// Preemptively grab the statsig material the capture harness stashed (Grok
// only; other sites don't install it, so this is `None`).
let maybe_statsig = read_captured_statsig(webview_window);
if let Some(statsig) = &maybe_statsig {
info!("Captured {} statsig for {} {}", website, statsig.method, statsig.path);
if maybe_statsig.is_some() {
info!("Captured {} statsig material (seed)", website);
}
let credential = save_web_credential(

View File

@ -10,8 +10,8 @@
//! makes on its own (no generation is triggered), so the stored statsig is a
//! session sample, not necessarily the video endpoint's.
use grok_consumer_statsig::statsig_cache_file::{decode_statsig, CapturedStatsig};
use grok_consumer_statsig::MINT_HARNESS_SCRIPT;
use grok_consumer_statsig::statsig_cache_file::decode_statsig;
use grok_consumer_statsig::{StatsigMaterial, MINT_HARNESS_SCRIPT};
use tauri::WebviewWindow;
/// Cookie the capture harness stashes the latest statsig report into. Skipped
@ -43,8 +43,13 @@ pub fn grok_statsig_init_script() -> String {
)
}
/// Read and decode the freshest statsig the harness captured, if any.
pub fn read_captured_statsig(webview: &WebviewWindow) -> Option<CapturedStatsig> {
/// Read the freshest statsig the harness captured and reduce it to reusable
/// [`StatsigMaterial`] (the seed), if any.
///
/// The captured signature yields the seed (decodable) but not the genuine-hex
/// fingerprint (SHA-locked), so the material is the seed alone — see
/// [`StatsigMaterial::generate_statsig`].
pub fn read_captured_statsig(webview: &WebviewWindow) -> Option<StatsigMaterial> {
let packed = webview
.cookies()
.ok()?
@ -55,15 +60,17 @@ pub fn read_captured_statsig(webview: &WebviewWindow) -> Option<CapturedStatsig>
parse_packed_statsig(&packed)
}
/// Parse the `method|path|statsigId|capturedAtMs` cookie payload and decode it.
fn parse_packed_statsig(packed: &str) -> Option<CapturedStatsig> {
/// Parse the `method|path|statsigId|capturedAtMs` cookie payload and reduce it
/// to the reusable seed material.
fn parse_packed_statsig(packed: &str) -> Option<StatsigMaterial> {
let mut parts = packed.splitn(4, '|');
let method = parts.next()?;
let path = parts.next()?;
let statsig_id = parts.next()?;
let captured_at_ms: i64 = parts.next()?.parse().ok()?;
decode_statsig(statsig_id, method, path, captured_at_ms / 1000).ok()
let captured = decode_statsig(statsig_id, method, path, captured_at_ms / 1000).ok()?;
Some(StatsigMaterial::from_seed_b64(captured.seed_b64))
}
#[cfg(test)]
@ -84,15 +91,11 @@ mod tests {
}
#[test]
fn parses_a_packed_capture_into_pieces() {
fn parses_a_packed_capture_into_seed_material() {
let packed = format!("POST|/rest/app-chat/conversations/new|{CAPTURE}|1787535100000");
let decoded = parse_packed_statsig(&packed).expect("should decode");
assert_eq!(decoded.method, "POST");
assert_eq!(decoded.path, "/rest/app-chat/conversations/new");
assert_eq!(decoded.statsig_id, CAPTURE);
assert_eq!(decoded.captured_at_unix, 1_787_535_100);
assert_eq!(decoded.key, 93);
assert!(!decoded.seed_b64.is_empty());
let material = parse_packed_statsig(&packed).expect("should decode");
// The reusable piece is the decoded seed (48 bytes -> 64 base64 chars).
assert_eq!(material.seed_b64.len(), 64);
}
#[test]