diff --git a/crates/native/src/backend.rs b/crates/native/src/backend.rs index 4ff9bd5..9f2eea5 100644 --- a/crates/native/src/backend.rs +++ b/crates/native/src/backend.rs @@ -14,6 +14,7 @@ use artcraft_launcher_core::protocol::{ use artcraft_launcher_core::release::{Asset, Release}; use artcraft_launcher_core::{CraftApp, Installed, PackageKind, SUITE, version}; +use crate::feed_throttle::{self, FeedThrottle, Outcome}; use crate::http::{self, Http, Stop}; use crate::install::{self, Job}; use crate::manifest::Manifest; @@ -28,9 +29,12 @@ const FRESH_FOR: i64 = 4 * 60; const GITHUB_FALLBACK_EVERY: i64 = 60 * 60; /// Until every app has loaded once, GitHub may be asked a little more eagerly. const EARLY_GITHUB_EVERY: i64 = 5 * 60; -/// Tries at the feed per check (after the very first poll), and the waits between them. +/// Tries at the feed per check (after the very first poll). The pause between tries is the +/// [`FeedThrottle`]'s: seconds during the first load, so a check retries; a minute or more after +/// that, so a check then tries once and the retries come from the next checks. const FEED_ATTEMPTS: usize = 3; -const FEED_RETRY_WAITS: [Duration; 2] = [Duration::from_secs(1), Duration::from_secs(3)]; +/// The longest a check waits on the [`FeedThrottle`] before carrying on without the feed. +const MAX_THROTTLE_WAIT: i64 = 10; /// After everything has loaded, the feed counts as seriously failed (and GitHub takes over) /// only after this many checks in a row without an answer… const SERIOUS_FEED_FAILURES: u32 = 3; @@ -65,6 +69,8 @@ struct Shared { feed_failures: Mutex, /// When the feed last answered (Unix seconds). feed_ok_at: Mutex>, + /// How often ArtCraft's API may be called. + feed_throttle: Mutex, } impl Shared { @@ -117,6 +123,7 @@ impl NativeBackend { loaded: AtomicBool::new(false), feed_failures: Mutex::new(0), feed_ok_at: Mutex::new(None), + feed_throttle: Mutex::new(FeedThrottle::load(&paths.feed_throttle(), crate::now())), last_check: Mutex::new(None), }); let backend = Self { shared, paths, http, source, platform: native_platform(), self_install: crate::self_update::current() }; @@ -496,20 +503,25 @@ fn feed_and_fallback(http: &Http, paths: &Paths, shared: &Shared) -> (Option { + *failures = 0; + *shared.feed_ok_at.lock().unwrap_or_else(PoisonError::into_inner) = Some(now); + (Some(feed), true, 0) + } + FeedFetch::Failed => { + *failures = failures.saturating_add(1); + (None, false, *failures) + } + // Not asked this time: things stand as the last request left them. + FeedFetch::Held => (None, *failures == 0 && saved.is_some(), *failures), } - *failures }; let ok_at = *shared.feed_ok_at.lock().unwrap_or_else(PoisonError::into_inner); - let answered = fresh.is_some(); let fallback = decide_fallback(answered, loaded, first_time, failures, ok_at, now); (fresh.or(saved), fallback) } @@ -565,28 +577,68 @@ fn feed_repo(feed: Option<&Feed>, app: &CraftApp) -> String { feed.map_or_else(|| app.repo.to_string(), |f| f.repo_for(app)) } -/// Asks ArtCraft's release feed up to `attempts` times (waiting 1 s, then 3 s, between tries), -/// saving a good answer. `None` when it never answered with something readable. -fn fetch_feed(http: &Http, paths: &Paths, attempts: usize) -> Option { - let url = feed_url(); +/// What came of asking the feed during one check. +enum FeedFetch { + Answered(Feed), + /// Asked, without a good answer. + Failed, + /// Not asked: the [`FeedThrottle`] said to hold off. + Held, +} + +/// Asks ArtCraft's release feed up to `attempts` times, as often as `throttle` allows (waiting at +/// most [`MAX_THROTTLE_WAIT`] for it), saving a good answer. +fn fetch_feed(http: &Http, paths: &Paths, url: &str, throttle: &Mutex, attempts: usize) -> FeedFetch { + let mut asked = false; for attempt in 0..attempts.max(1) { - if attempt > 0 { - std::thread::sleep(FEED_RETRY_WAITS.get(attempt - 1).copied().unwrap_or(Duration::from_secs(3))); + let wait = throttle.lock().unwrap_or_else(PoisonError::into_inner).wait(crate::now()); + if wait > MAX_THROTTLE_WAIT { + if !asked { + log::info!("holding off the ArtCraft API for {wait} s"); + } + break; } - match http.get_feed(&url) { + if wait > 0 { + std::thread::sleep(Duration::from_secs(wait.unsigned_abs())); + } + { + let mut throttle = throttle.lock().unwrap_or_else(PoisonError::into_inner); + let now = crate::now(); + if throttle.wait(now) > 0 { + break; + } + throttle.sending(now); + } + asked = true; + let outcome = match http.get_feed(url) { Ok(body) => match Feed::parse(&body) { Some(feed) => { if let Err(e) = crate::paths::write_atomic(&paths.feed_cache(), body.as_bytes()) { log::warn!("couldn't save the release feed: {e}"); } - return Some(feed); + throttle.lock().unwrap_or_else(PoisonError::into_inner).finished(Outcome::Answered, crate::now(), 0.0); + return FeedFetch::Answered(feed); + } + None => { + log::warn!("the release feed answered with something unreadable (try {})", attempt + 1); + Outcome::Failed { retry_after: None } } - None => log::warn!("the release feed answered with something unreadable (try {})", attempt + 1), }, - Err(e) => log::info!("release feed unavailable (try {}): {e}", attempt + 1), + Err(e) if e.throttled => { + log::warn!("the ArtCraft API asked us to slow down (HTTP 429); holding off"); + Outcome::Throttled { retry_after: e.retry_after } + } + Err(e) => { + log::info!("release feed unavailable (try {}): {}", attempt + 1, e.why); + Outcome::Failed { retry_after: e.retry_after } + } + }; + throttle.lock().unwrap_or_else(PoisonError::into_inner).finished(outcome, crate::now(), feed_throttle::jitter()); + if matches!(outcome, Outcome::Throttled { .. }) { + break; } } - None + if asked { FeedFetch::Failed } else { FeedFetch::Held } } /// The feed's address. `ARTCRAFT_LAUNCHER_FEED_URL` points at a staging server or a local copy @@ -755,9 +807,63 @@ mod tests { loaded: AtomicBool::new(false), feed_failures: Mutex::new(0), feed_ok_at: Mutex::new(None), + feed_throttle: Mutex::new(FeedThrottle::new()), } } + /// A local server answering every request with `status` (and `Retry-After: 120` on a 429), + /// counting the requests. + fn serve(status: &'static str) -> (String, Arc) { + use std::io::{Read, Write}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/v1/craft_apps/release_info", listener.local_addr().unwrap()); + let hits = Arc::new(AtomicUsize::new(0)); + let counted = hits.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { continue }; + let mut buf = [0u8; 4096]; + let _ = stream.read(&mut buf); + counted.fetch_add(1, Ordering::SeqCst); + let extra = if status.starts_with("429") { "Retry-After: 120\r\n" } else { "" }; + let _ = write!(stream, "HTTP/1.1 {status}\r\n{extra}Content-Length: 0\r\nConnection: close\r\n\r\n"); + } + }); + (url, hits) + } + + #[test] + fn a_429_stops_the_retries_and_holds_off_the_api() { + let (url, hits) = serve("429 Too Many Requests"); + let dir = tempfile::tempdir().unwrap(); + let paths = Paths::under(dir.path()); + let http = Http::new().unwrap(); + let throttle = Mutex::new(FeedThrottle::load(&paths.feed_throttle(), crate::now())); + assert!(matches!(fetch_feed(&http, &paths, &url, &throttle, FEED_ATTEMPTS), FeedFetch::Failed)); + assert_eq!(hits.load(Ordering::SeqCst), 1); + // The next checks (and a relaunch) don't call it at all. + assert!(matches!(fetch_feed(&http, &paths, &url, &throttle, FEED_ATTEMPTS), FeedFetch::Held)); + let relaunched = Mutex::new(FeedThrottle::load(&paths.feed_throttle(), crate::now())); + assert!(matches!(fetch_feed(&http, &paths, &url, &relaunched, FEED_ATTEMPTS), FeedFetch::Held)); + assert_eq!(hits.load(Ordering::SeqCst), 1); + } + + #[test] + fn a_failing_api_is_retried_quickly_only_during_the_first_load() { + let (url, hits) = serve("503 Service Unavailable"); + let dir = tempfile::tempdir().unwrap(); + let paths = Paths::under(dir.path()); + let http = Http::new().unwrap(); + let throttle = Mutex::new(FeedThrottle::new()); + // First load: three tries, a couple of seconds apart. + assert!(matches!(fetch_feed(&http, &paths, &url, &throttle, FEED_ATTEMPTS), FeedFetch::Failed)); + assert_eq!(hits.load(Ordering::SeqCst), 3); + // Once the server has answered, never more than one a minute. + throttle.lock().unwrap().finished(Outcome::Answered, crate::now(), 0.0); + assert!(matches!(fetch_feed(&http, &paths, &url, &throttle, FEED_ATTEMPTS), FeedFetch::Held)); + assert_eq!(hits.load(Ordering::SeqCst), 3); + } + const NOW: i64 = 1_791_500_000; /// The steady state: everything has loaded and the feed is answering. const LOADED: Fallback = Fallback { trust_feed: true, github_every: GITHUB_FALLBACK_EVERY }; diff --git a/crates/native/src/feed_throttle.rs b/crates/native/src/feed_throttle.rs new file mode 100644 index 0000000..6f4e60f --- /dev/null +++ b/crates/native/src/feed_throttle.rs @@ -0,0 +1,378 @@ +//! [`FeedThrottle`]: how often the launcher may call ArtCraft's API (the release feed). +//! +//! Every launcher in the world polls the same server, so the limits are strict and favour the +//! server over freshness (the last saved answer always stands in): +//! +//! - **At most one request a minute** once the first load is over. +//! - **During the first load** (until the feed first answers, at most [`WARM_REQUESTS`] +//! requests), up to ten a minute: front-loaded, with the gaps doubling after each failure +//! (2 s, 4 s, 8 s, …), so a struggling server sees each client ease off quickly. +//! - **Failures back off**: after a failed request the next waits 2, 4, 8 … minutes (at most an +//! hour), with jitter. +//! - **HTTP 429 throttles hard**: at least 15 minutes, doubling with each 429 in a row up to six +//! hours, never sooner than the server's `Retry-After`, plus up to 50% jitter so a herd of +//! clients doesn't come back in step. This survives restarts (saved to disk), so relaunching +//! doesn't skip the wait. + +use std::collections::VecDeque; +use std::hash::{BuildHasher, Hasher}; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +/// The window the per-minute limits count over (seconds). +const WINDOW: i64 = 60; +/// Requests per [`WINDOW`] once the first load is over. +const STEADY_PER_WINDOW: usize = 1; +/// Requests per [`WINDOW`] during the first load. +const WARM_PER_WINDOW: usize = 10; +/// The first load ends at the first good answer, or after this many requests. +const WARM_REQUESTS: u32 = 10; +/// Gaps between first-load requests after 0, 1, 2 … failures in a row (seconds). +const WARM_GAPS: [i64; 6] = [1, 2, 4, 8, 16, 32]; +/// After a failure (once loaded), the wait before the next request: this, doubled per failure in +/// a row… +const FAILURE_BACKOFF: i64 = 2 * 60; +/// …up to this. +const MAX_FAILURE_BACKOFF: i64 = 60 * 60; +/// After an HTTP 429, the wait before the next request: this, doubled per 429 in a row… +const THROTTLE_BACKOFF: i64 = 15 * 60; +/// …up to this. +const MAX_THROTTLE_BACKOFF: i64 = 6 * 60 * 60; +/// A `Retry-After` beyond this is ignored as nonsense (we still wait our own backoff). +const MAX_RETRY_AFTER: i64 = 24 * 60 * 60; + +/// What happened to one request, as far as pacing is concerned. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Outcome { + /// A readable answer. + Answered, + /// No answer, an error status or an unreadable body: the server may be struggling. + /// `retry_after` is the server's `Retry-After` (seconds), if it sent one. + Failed { retry_after: Option }, + /// HTTP 429 Too Many Requests. + Throttled { retry_after: Option }, +} + +/// The part that outlives the process, so a relaunch keeps backing off. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +struct Saved { + /// No request before this (Unix seconds). + not_before: i64, + /// Failed requests in a row. + failures: u32, + /// HTTP 429s in a row. + throttles: u32, +} + +/// The pacing state for ArtCraft's API. Times are Unix seconds, passed in so it can be tested. +#[derive(Debug)] +pub struct FeedThrottle { + saved: Saved, + /// When recent requests were sent (the last [`WINDOW`] seconds' worth). + sent: VecDeque, + /// Requests sent during the first load. + warm_sent: u32, + /// The first load is over: the feed has answered, or we've used up the first-load requests. + warmed: bool, + /// Where the state is saved, if anywhere. + file: Option, +} + +impl FeedThrottle { + /// Fresh state, not saved anywhere. + pub fn new() -> Self { + Self { saved: Saved::default(), sent: VecDeque::new(), warm_sent: 0, warmed: false, file: None } + } + + /// The state saved at `file` by an earlier run (if readable and sane), saved back there as it + /// changes. A missing or malformed file just means no backoff is pending. + pub fn load(file: &Path, now: i64) -> Self { + let mut saved = std::fs::read(file).ok().and_then(|b| serde_json::from_slice::(&b).ok()).unwrap_or_default(); + // A wait far in the future means a broken file or a clock that jumped back: drop it. + if saved.not_before > now + MAX_THROTTLE_BACKOFF * 2 { + saved.not_before = 0; + } + Self { saved, file: Some(file.to_path_buf()), ..Self::new() } + } + + /// Seconds to wait before the next request may be sent (0: now). + pub fn wait(&self, now: i64) -> i64 { + let warming = !self.warmed; + let per_window = if warming { WARM_PER_WINDOW } else { STEADY_PER_WINDOW }; + let mut at = self.saved.not_before; + let recent: Vec = self.sent.iter().copied().filter(|t| now - t < WINDOW).collect(); + if recent.len() >= per_window + && let Some(oldest) = recent.get(recent.len() - per_window) + { + at = at.max(oldest + WINDOW); + } + if warming && let Some(last) = recent.last() { + let gap = WARM_GAPS.get(self.saved.failures as usize).or(WARM_GAPS.last()).copied().unwrap_or(WINDOW); + at = at.max(last + gap); + } + (at - now).max(0) + } + + /// Notes that a request is being sent now. + pub fn sending(&mut self, now: i64) { + self.sent.push_back(now); + while self.sent.front().is_some_and(|t| now - t >= WINDOW) { + self.sent.pop_front(); + } + if !self.warmed { + self.warm_sent += 1; + if self.warm_sent >= WARM_REQUESTS { + self.warmed = true; + } + } + } + + /// Notes how the request went. `jitter` is in `0.0..1.0` (see [`jitter`]). + pub fn finished(&mut self, outcome: Outcome, now: i64, jitter: f64) { + let before = self.saved; + match outcome { + Outcome::Answered => { + self.warmed = true; + self.saved = Saved::default(); + } + Outcome::Failed { retry_after } => { + self.saved.failures = self.saved.failures.saturating_add(1); + self.saved.throttles = 0; + // During the first load the doubling gaps in `wait` do the backing off. + let backoff = if self.warmed { doubled(FAILURE_BACKOFF, self.saved.failures, MAX_FAILURE_BACKOFF) } else { 0 }; + let backoff = spread(backoff, 0.4 * jitter - 0.2); + self.saved.not_before = now + backoff.max(sane(retry_after)); + } + Outcome::Throttled { retry_after } => { + self.warmed = true; + self.saved.throttles = self.saved.throttles.saturating_add(1); + let backoff = doubled(THROTTLE_BACKOFF, self.saved.throttles, MAX_THROTTLE_BACKOFF); + // Only ever later than asked: spreading the herd out, never bringing it back early. + self.saved.not_before = now + spread(backoff.max(sane(retry_after)), 0.5 * jitter); + } + } + if self.saved != before { + self.save(); + } + } + + /// The server sent HTTP 429 recently and we're still waiting it out. + pub fn throttled(&self, now: i64) -> bool { + self.saved.throttles > 0 && self.saved.not_before > now + } + + fn save(&self) { + let Some(file) = &self.file else { return }; + let saved = serde_json::to_vec(&self.saved).unwrap_or_default(); + if let Err(e) = crate::paths::write_atomic(file, &saved) { + log::warn!("couldn't save the ArtCraft API backoff: {e}"); + } + } +} + +impl Default for FeedThrottle { + fn default() -> Self { + Self::new() + } +} + +/// `base` doubled for each repeat after the first (`times` ≥ 1), at most `max`. +fn doubled(base: i64, times: u32, max: i64) -> i64 { + let doublings = times.saturating_sub(1).min(16); + base.saturating_mul(1 << doublings).min(max) +} + +/// `secs` scaled by `1 + by`. +fn spread(secs: i64, by: f64) -> i64 { + #[allow(clippy::cast_possible_truncation, clippy::cast_precision_loss)] + let scaled = (secs as f64 * (1.0 + by)).round() as i64; + scaled.max(0) +} + +/// A `Retry-After` in seconds, ignoring negative or absurd values. +fn sane(retry_after: Option) -> i64 { + retry_after.filter(|s| (0..=MAX_RETRY_AFTER).contains(s)).unwrap_or(0) +} + +/// A random number in `0.0..1.0`, different in every launcher (std's hasher keys are random). +pub fn jitter() -> f64 { + let bits = std::collections::hash_map::RandomState::new().build_hasher().finish(); + #[allow(clippy::cast_precision_loss)] + let fraction = (bits >> 11) as f64 / (1u64 << 53) as f64; + fraction +} + +#[cfg(test)] +mod tests { + use super::*; + + const NOW: i64 = 1_791_500_000; + + /// Sends requests as fast as the throttle allows for `secs`, each failing with `outcome`. + /// Returns when they were sent. + fn hammer(t: &mut FeedThrottle, from: i64, secs: i64, outcome: Outcome) -> Vec { + let mut sent = Vec::new(); + let mut now = from; + while now < from + secs { + let wait = t.wait(now); + if wait > 0 { + now += wait; + continue; + } + t.sending(now); + sent.push(now); + t.finished(outcome, now, 0.5); + } + sent.retain(|s| *s < from + secs); + sent + } + + fn per_minute_max(sent: &[i64]) -> usize { + sent.iter().map(|s| sent.iter().filter(|t| **t >= *s && **t < s + WINDOW).count()).max().unwrap_or(0) + } + + #[test] + fn once_loaded_never_more_than_one_request_a_minute() { + let mut t = FeedThrottle::new(); + t.sending(NOW); + t.finished(Outcome::Answered, NOW, 0.5); + let sent = hammer(&mut t, NOW, 3600, Outcome::Answered); + assert_eq!(per_minute_max(&sent), 1); + assert_eq!(sent.len(), 59); + assert_eq!(sent.first(), Some(&(NOW + 60))); + } + + #[test] + fn the_first_load_may_go_faster_but_never_past_ten_a_minute() { + let mut t = FeedThrottle::new(); + let sent = hammer(&mut t, NOW, 60, Outcome::Failed { retry_after: None }); + assert!(sent.len() > 1 && per_minute_max(&sent) <= 10, "{sent:?}"); + // Front-loaded and falling off: the gaps grow with each failure. + let gaps: Vec = sent.windows(2).map(|w| w[1] - w[0]).collect(); + assert!(gaps.windows(2).all(|g| g[1] >= g[0]), "{gaps:?}"); + assert_eq!(sent.first(), Some(&NOW)); + } + + #[test] + fn a_healthy_first_load_retries_quickly() { + // Even with every request succeeding, after the first answer it's one a minute. + let mut t = FeedThrottle::new(); + let sent = hammer(&mut t, NOW, 120, Outcome::Answered); + assert_eq!(sent, vec![NOW, NOW + 60]); + } + + #[test] + fn the_first_load_ends_after_ten_requests_even_without_an_answer() { + let mut t = FeedThrottle::new(); + for i in 0..WARM_REQUESTS { + t.sending(NOW + i64::from(i)); + } + // Back to one a minute. + assert!(t.wait(NOW + 10) >= 50); + } + + #[test] + fn failures_back_off_after_loading() { + let mut t = FeedThrottle::new(); + t.sending(NOW); + t.finished(Outcome::Answered, NOW, 0.5); + let sent = hammer(&mut t, NOW + 60, 6 * 3600, Outcome::Failed { retry_after: None }); + let gaps: Vec = sent.windows(2).map(|w| w[1] - w[0]).collect(); + assert_eq!(gaps.first(), Some(&(2 * 60))); + assert!(gaps.windows(2).all(|g| g[1] >= g[0]), "{gaps:?}"); + assert_eq!(gaps.last(), Some(&MAX_FAILURE_BACKOFF)); + } + + #[test] + fn a_good_answer_resets_the_backoff() { + let mut t = FeedThrottle::new(); + t.warmed = true; + for i in 0..5 { + t.finished(Outcome::Failed { retry_after: None }, NOW + i, 0.5); + } + t.finished(Outcome::Answered, NOW + 10, 0.5); + assert_eq!(t.wait(NOW + 10), 0); + } + + #[test] + fn a_429_throttles_hard_and_harder() { + let mut t = FeedThrottle::new(); + t.sending(NOW); + t.finished(Outcome::Throttled { retry_after: None }, NOW, 0.0); + assert_eq!(t.wait(NOW), THROTTLE_BACKOFF); + assert!(t.throttled(NOW)); + let sent = hammer(&mut t, NOW, 24 * 3600, Outcome::Throttled { retry_after: None }); + let gaps: Vec = sent.windows(2).map(|w| w[1] - w[0]).collect(); + assert!(gaps.windows(2).all(|g| g[1] >= g[0]), "{gaps:?}"); + assert!(gaps.iter().all(|g| *g >= THROTTLE_BACKOFF)); + assert!(gaps.last().is_some_and(|g| *g >= MAX_THROTTLE_BACKOFF)); + } + + #[test] + fn a_429_during_the_first_load_ends_the_fast_start() { + let mut t = FeedThrottle::new(); + t.sending(NOW); + t.finished(Outcome::Throttled { retry_after: None }, NOW, 0.0); + let sent = hammer(&mut t, NOW + 1, 3600, Outcome::Throttled { retry_after: None }); + assert!(sent.iter().all(|s| *s >= NOW + THROTTLE_BACKOFF), "{sent:?}"); + } + + #[test] + fn retry_after_is_honoured_but_never_shortens_our_wait() { + let mut t = FeedThrottle::new(); + t.finished(Outcome::Throttled { retry_after: Some(3 * 3600) }, NOW, 0.0); + assert_eq!(t.wait(NOW), 3 * 3600); + let mut t = FeedThrottle::new(); + t.finished(Outcome::Throttled { retry_after: Some(5) }, NOW, 0.0); + assert_eq!(t.wait(NOW), THROTTLE_BACKOFF); + // Nonsense is ignored. + let mut t = FeedThrottle::new(); + t.finished(Outcome::Throttled { retry_after: Some(i64::MAX) }, NOW, 0.0); + assert_eq!(t.wait(NOW), THROTTLE_BACKOFF); + // A 503 with Retry-After is honoured too. + let mut t = FeedThrottle::new(); + t.finished(Outcome::Failed { retry_after: Some(600) }, NOW, 0.5); + assert_eq!(t.wait(NOW), 600); + } + + #[test] + fn jitter_only_ever_delays_a_429() { + for j in [0.0, 0.25, 0.5, 0.99] { + let mut t = FeedThrottle::new(); + t.finished(Outcome::Throttled { retry_after: None }, NOW, j); + let wait = t.wait(NOW); + assert!((THROTTLE_BACKOFF..=THROTTLE_BACKOFF * 3 / 2).contains(&wait), "{j}: {wait}"); + } + let j = jitter(); + assert!((0.0..1.0).contains(&j)); + } + + #[test] + fn the_backoff_survives_a_restart() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("feed-throttle.json"); + let mut t = FeedThrottle::load(&file, NOW); + t.finished(Outcome::Throttled { retry_after: None }, NOW, 0.0); + let again = FeedThrottle::load(&file, NOW + 60); + assert_eq!(again.wait(NOW + 60), THROTTLE_BACKOFF - 60); + assert!(again.throttled(NOW + 60)); + // A good answer clears it. + let mut again = again; + again.finished(Outcome::Answered, NOW + THROTTLE_BACKOFF, 0.0); + assert_eq!(FeedThrottle::load(&file, NOW + THROTTLE_BACKOFF).wait(NOW + THROTTLE_BACKOFF), 0); + } + + #[test] + fn a_broken_or_absurd_saved_file_means_no_wait() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("feed-throttle.json"); + std::fs::write(&file, b"{not json").unwrap(); + assert_eq!(FeedThrottle::load(&file, NOW).wait(NOW), 0); + std::fs::write(&file, br#"{"not_before": 99999999999, "throttles": 3}"#).unwrap(); + assert_eq!(FeedThrottle::load(&file, NOW).wait(NOW), 0); + std::fs::write(&file, br#"{"unknown": true}"#).unwrap(); + assert_eq!(FeedThrottle::load(&file, NOW).wait(NOW), 0); + } +} diff --git a/crates/native/src/http.rs b/crates/native/src/http.rs index 506a143..ed8cbfb 100644 --- a/crates/native/src/http.rs +++ b/crates/native/src/http.rs @@ -39,6 +39,16 @@ pub struct Http { token: Option, } +/// Why ArtCraft's release feed couldn't be read. +#[derive(Debug)] +pub struct FeedError { + pub why: String, + /// The server answered HTTP 429 Too Many Requests. + pub throttled: bool, + /// The server's `Retry-After`, in seconds. + pub retry_after: Option, +} + impl Http { pub fn new() -> Result { let roots = os_roots()?; @@ -121,11 +131,15 @@ impl Http { } /// ArtCraft's release feed, or why it couldn't be read. Never takes more than ~10 seconds. - pub fn get_feed(&self, url: &str) -> Result { - let mut resp = self.feed.get(url).header("Accept", "application/json").call().map_err(|e| format!("couldn't reach {} ({e})", host(url)))?; + pub fn get_feed(&self, url: &str) -> Result { + let failed = |why: String| FeedError { why, throttled: false, retry_after: None }; + let mut resp = + self.feed.get(url).header("Accept", "application/json").call().map_err(|e| failed(format!("couldn't reach {} ({e})", host(url))))?; + // Seconds only; an HTTP date falls back to our own backoff. + let retry_after = resp.headers().get("retry-after").and_then(|v| v.to_str().ok()).and_then(|r| r.trim().parse::().ok()); match resp.status().as_u16() { - 200 => resp.body_mut().with_config().limit(MAX_SMALL).read_to_string().map_err(|e| format!("unreadable answer ({e})")), - s => Err(format!("{} answered HTTP {s}", host(url))), + 200 => resp.body_mut().with_config().limit(MAX_SMALL).read_to_string().map_err(|e| failed(format!("unreadable answer ({e})"))), + s => Err(FeedError { why: format!("{} answered HTTP {s}", host(url)), throttled: s == 429, retry_after }), } } diff --git a/crates/native/src/lib.rs b/crates/native/src/lib.rs index dd59f0e..3787e50 100644 --- a/crates/native/src/lib.rs +++ b/crates/native/src/lib.rs @@ -9,6 +9,7 @@ pub mod backend; pub mod detect; +pub mod feed_throttle; pub mod http; pub mod install; pub mod login; diff --git a/crates/native/src/paths.rs b/crates/native/src/paths.rs index ee9920e..7698221 100644 --- a/crates/native/src/paths.rs +++ b/crates/native/src/paths.rs @@ -41,6 +41,11 @@ impl Paths { self.cache.join("feed.json") } + /// How long to hold off calling ArtCraft's API (after failures or HTTP 429), across restarts. + pub fn feed_throttle(&self) -> PathBuf { + self.cache.join("feed-throttle.json") + } + pub fn manifest(&self) -> PathBuf { self.data.join("installed.json") }