445 lines
18 KiB
Rust
445 lines
18 KiB
Rust
//! Portuguese spelling suggestions, best first.
|
|
//!
|
|
//! Candidates come in rounds of rising cost, each checked against the dictionary:
|
|
//! 1. letter case (`brasília` → `Brasília`);
|
|
//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`coraçao` → `coração`,
|
|
//! `saude` → `saúde`), up to four letters changed — Portuguese typed without accents is the
|
|
//! most common slip, so adding one ranks first, while taking away an accent the writer typed
|
|
//! ranks after the known confusions;
|
|
//! 3. the dictionary's `REP` table of common confusions (`seje` → `seja`, `excessão` →
|
|
//! `exceção`, `previlégio` → `privilégio`);
|
|
//! 4. a missing space (`nãoé` → `não é`, `paraela` → `para ela`), cheaper when one half is a
|
|
//! short function word;
|
|
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
|
|
//! 6. when that finds fewer than asked for: the stems that start like the word (accents
|
|
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
|
|
//! in which an accent slip costs half (two typos at most).
|
|
//!
|
|
//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand
|
|
//! dictionary lookups at most.
|
|
|
|
use std::collections::{HashMap, HashSet};
|
|
|
|
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
|
|
|
/// Short words often run into the next one (`não é`, `da água`, `para isso`).
|
|
const LEADING_WORDS: &[&str] = &[
|
|
"a", "o", "e", "é", "de", "do", "da", "dos", "das", "em", "no", "na", "nos", "nas", "por", "para", "com", "sem", "sob", "sobre", "ao", "aos",
|
|
"à", "às", "que", "se", "não", "como", "mas", "mais", "já", "até", "entre", "contra", "perante", "desde", "após", "ante", "durante", "mediante",
|
|
"segundo", "conforme", "salvo", "fora", "onde", "quando", "bem", "mal", "pois", "logo", "portanto", "porém", "todavia", "contudo", "assim",
|
|
"tão", "tanto", "tudo", "nada", "algo", "cada", "outro", "mesmo", "próprio", "ainda", "tampouco", "quase", "talvez", "nunca", "sempre",
|
|
];
|
|
/// Short words often run into the one before (`amo-te`, `deu-lhe`; in writing without the
|
|
/// hyphen: `amote`, `deulhe`).
|
|
const TRAILING_WORDS: &[&str] = &[
|
|
"me", "te", "se", "lhe", "nos", "vos", "lhes", "o", "a", "os", "as", "ele", "ela", "eles", "elas", "isso", "isto", "aquilo", "mim", "ti", "si",
|
|
"eu", "tu", "que", "de", "em", "é",
|
|
];
|
|
const MAX_LETTERS: usize = 40;
|
|
const MAX_MAP_CHANGES: usize = 4;
|
|
const MAX_MAP_VARIANTS: usize = 4096;
|
|
/// Stems the scan of round 6 expands.
|
|
const SCAN_STEMS: usize = 48;
|
|
|
|
const COST_CASE: u32 = 0;
|
|
const COST_ADD_DIACRITIC: u32 = 4;
|
|
const COST_OTHER_MAP: u32 = 7;
|
|
const COST_DROP_DIACRITIC: u32 = 11;
|
|
const COST_REP: u32 = 10;
|
|
const COST_SPLIT: u32 = 12;
|
|
const COST_SPLIT_RARE: u32 = 18;
|
|
const COST_SWAP: u32 = 14;
|
|
const COST_DROP_OR_ADD: u32 = 15;
|
|
const COST_MISTYPE: u32 = 16;
|
|
const COST_SCAN: u32 = 20;
|
|
|
|
struct Collector<'a> {
|
|
d: &'a AffixDict,
|
|
/// The word, lower case.
|
|
input: String,
|
|
tested: HashSet<String>,
|
|
/// Suggestion → (cost, order found).
|
|
found: HashMap<String, (u32, usize)>,
|
|
seq: usize,
|
|
}
|
|
|
|
impl Collector<'_> {
|
|
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
|
/// word is in the dictionary, as written or capitalized (names).
|
|
fn offer(&mut self, cand: &str, cost: u32) {
|
|
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
|
return;
|
|
}
|
|
let mut words = Vec::new();
|
|
let mut cost = cost;
|
|
for part in cand.split(' ') {
|
|
if self.d.contains(part) {
|
|
words.push(part.to_string());
|
|
} else {
|
|
let c = capitalize(part);
|
|
if !self.d.contains(&c) {
|
|
return;
|
|
}
|
|
words.push(c);
|
|
// A name ranks after a common word at the same distance (`moça`, then `Moça`).
|
|
cost += 1;
|
|
}
|
|
}
|
|
self.keep(words.join(" "), cost);
|
|
}
|
|
|
|
/// Keep a suggestion known to be correct.
|
|
fn keep(&mut self, text: String, cost: u32) {
|
|
self.seq += 1;
|
|
let seq = self.seq;
|
|
let e = self.found.entry(text).or_insert((cost, seq));
|
|
if cost < e.0 {
|
|
*e = (cost, seq);
|
|
}
|
|
}
|
|
|
|
fn count_within(&self, cost: u32) -> usize {
|
|
self.found.values().filter(|(c, _)| *c <= cost).count()
|
|
}
|
|
}
|
|
|
|
/// The MAP group of each letter (its other members are its likely confusions).
|
|
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
|
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
|
|
}
|
|
|
|
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
|
|
/// commonest slip (Portuguese typed without accents: `coracao`); an accent the writer did type
|
|
/// is likelier right, so taking one away costs more than a known confusion from the REP table
|
|
/// (`seje` → `seja` before `sejé`).
|
|
fn map_cost(from: char, to: char) -> u32 {
|
|
match (from.is_ascii(), to.is_ascii()) {
|
|
(true, false) => COST_ADD_DIACRITIC,
|
|
(false, true) => COST_DROP_DIACRITIC,
|
|
_ => COST_OTHER_MAP,
|
|
}
|
|
}
|
|
|
|
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
|
|
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
|
if out.len() >= MAX_MAP_VARIANTS {
|
|
return;
|
|
}
|
|
let Some(&c) = chars.get(i) else {
|
|
if changes > 0 {
|
|
out.push((buf.iter().collect(), cost));
|
|
}
|
|
return;
|
|
};
|
|
buf.push(c);
|
|
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
|
buf.pop();
|
|
if changes < MAX_MAP_CHANGES {
|
|
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
|
buf.push(a);
|
|
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
|
buf.pop();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`.
|
|
fn fold(c: char, map: &[Vec<char>]) -> char {
|
|
let l = c.to_lowercase().next().unwrap_or(c);
|
|
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
|
}
|
|
|
|
/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other
|
|
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
|
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
|
let m = b.len();
|
|
let over = limit + 1;
|
|
let mut prev2 = vec![0u32; m + 1];
|
|
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
|
let mut cur = vec![0u32; m + 1];
|
|
for (i, &ai) in a.iter().enumerate() {
|
|
if let Some(c) = cur.get_mut(0) {
|
|
*c = (i as u32 + 1) * 2;
|
|
}
|
|
let mut row_min = u32::MAX;
|
|
for (j, &bj) in b.iter().enumerate() {
|
|
let sub = if ai == bj {
|
|
0
|
|
} else if related(ai, bj) {
|
|
1
|
|
} else {
|
|
2
|
|
};
|
|
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
|
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
|
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
|
v = v.min(at(&prev2, j - 1) + 2);
|
|
}
|
|
if let Some(c) = cur.get_mut(j + 1) {
|
|
*c = v;
|
|
}
|
|
row_min = row_min.min(v);
|
|
}
|
|
if m > 0 && row_min > limit {
|
|
return over;
|
|
}
|
|
std::mem::swap(&mut prev2, &mut prev);
|
|
std::mem::swap(&mut prev, &mut cur);
|
|
}
|
|
prev.get(m).copied().unwrap_or(over).min(over)
|
|
}
|
|
|
|
/// Spelling suggestions for a Portuguese word, best first, at most `max`, in the word's letter case.
|
|
pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
|
suggest_in(&super::dictionary(), word, max)
|
|
}
|
|
|
|
/// [`suggest`] against a given dictionary.
|
|
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
|
let word = word.trim();
|
|
let n = word.chars().count();
|
|
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
|
return Vec::new();
|
|
}
|
|
let case = casing(word);
|
|
let input = word.to_lowercase();
|
|
let chars: Vec<char> = input.chars().collect();
|
|
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
|
// 1. Letter case.
|
|
if d.contains(&input) && word != input {
|
|
col.keep(input.clone(), COST_CASE);
|
|
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
|
|
col.keep(capitalize(&input), COST_CASE);
|
|
}
|
|
// 2. Accents and other MAP slips.
|
|
let map = d.map();
|
|
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
|
let mut variants = Vec::new();
|
|
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
|
|
variants.sort_by_key(|v| v.1);
|
|
for (v, cost) in &variants {
|
|
col.offer(v, *cost);
|
|
}
|
|
// 3. Common confusions.
|
|
for (from, to) in d.rep() {
|
|
if from.is_empty() {
|
|
continue;
|
|
}
|
|
for (i, _) in input.match_indices(from.as_str()) {
|
|
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
|
col.offer(&format!("{a}{to}{b}"), COST_REP);
|
|
}
|
|
}
|
|
}
|
|
// 4. A missing space: cheap after a short leading word or before a short trailing one,
|
|
// dearer between two longer words, never leaving a stray short half.
|
|
for (i, _) in input.char_indices().skip(1) {
|
|
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
|
let long = |p: &str| p.chars().count() >= 3;
|
|
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
|
COST_SPLIT
|
|
} else if long(a) && long(b) {
|
|
COST_SPLIT_RARE
|
|
} else {
|
|
continue;
|
|
};
|
|
col.offer(&format!("{a} {b}"), cost);
|
|
}
|
|
// 5. One typo.
|
|
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
|
|
if letters.is_empty() {
|
|
letters = "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw".chars().collect();
|
|
}
|
|
let s = |v: &[char]| v.iter().collect::<String>();
|
|
for i in 0..n.saturating_sub(1) {
|
|
if chars.get(i) != chars.get(i + 1) {
|
|
let mut v = chars.clone();
|
|
v.swap(i, i + 1);
|
|
col.offer(&s(&v), COST_SWAP);
|
|
}
|
|
}
|
|
for i in 0..n {
|
|
let mut v = chars.clone();
|
|
v.remove(i);
|
|
col.offer(&s(&v), COST_DROP_OR_ADD);
|
|
}
|
|
for i in 0..=n {
|
|
for &c in &letters {
|
|
let mut v = chars.clone();
|
|
v.insert(i, c);
|
|
col.offer(&s(&v), COST_DROP_OR_ADD);
|
|
}
|
|
}
|
|
for i in 0..n {
|
|
for &c in &letters {
|
|
if chars.get(i) != Some(&c) {
|
|
let mut v = chars.clone();
|
|
if let Some(x) = v.get_mut(i) {
|
|
*x = c;
|
|
}
|
|
col.offer(&s(&v), COST_MISTYPE);
|
|
}
|
|
}
|
|
}
|
|
// 6. Further off: scan the stems that start like the word.
|
|
if col.count_within(COST_MISTYPE) < max {
|
|
scan(&mut col, &chars);
|
|
}
|
|
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
|
// Among equal costs, the change nearest the word's start is the likelier intent
|
|
// (`saúde` before `saudê`: both add an accent, but at the first difference).
|
|
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
|
|
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
|
|
let mut out: Vec<String> = Vec::new();
|
|
for (text, _) in found {
|
|
let t = match case {
|
|
Casing::Initial => capitalize(&text),
|
|
Casing::Upper if n > 1 => text.to_uppercase(),
|
|
_ => text,
|
|
};
|
|
if t != word && !out.contains(&t) {
|
|
out.push(t);
|
|
}
|
|
if out.len() >= max {
|
|
break;
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
|
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
|
fn scan(col: &mut Collector, chars: &[char]) {
|
|
let d = col.d;
|
|
let map = d.map();
|
|
let Some(&first) = chars.first() else { return };
|
|
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
|
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
|
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
|
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
|
firsts.extend(upper);
|
|
firsts.sort_unstable();
|
|
firsts.dedup();
|
|
let n = folded.len();
|
|
let mut best: Vec<(u32, &str)> = Vec::new();
|
|
for f in firsts {
|
|
let prefix = f.to_string();
|
|
for stem in d.stems_with_prefix(&prefix) {
|
|
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
|
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
|
if shared < 2 {
|
|
continue;
|
|
}
|
|
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
|
// beyond what an ending explains.
|
|
let extra = sf.len().saturating_sub(shared);
|
|
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
|
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
|
if score == 0 {
|
|
continue;
|
|
}
|
|
if best.len() < SCAN_STEMS {
|
|
best.push((score, stem));
|
|
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
|
&& score > min.0
|
|
{
|
|
*min = (score, stem);
|
|
}
|
|
}
|
|
}
|
|
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
|
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
|
let limit = 4;
|
|
for (_, stem) in best {
|
|
for form in d.forms(stem) {
|
|
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
|
if lf.len().abs_diff(chars.len()) > 2 {
|
|
continue;
|
|
}
|
|
let dist = weighted_distance(&lf, chars, &related, limit);
|
|
if dist <= limit && lf.as_slice() != chars {
|
|
col.keep(form, COST_SCAN + dist * 3);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn has(word: &str, want: &str) {
|
|
let s = suggest(word, 8);
|
|
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
|
|
}
|
|
|
|
#[test]
|
|
fn portuguese_misspellings_get_the_right_suggestion() {
|
|
crate::pt::test_dict::install();
|
|
has("coraçao", "coração");
|
|
has("saude", "saúde");
|
|
has("seje", "seja");
|
|
has("excessão", "exceção");
|
|
has("previlégio", "privilégio");
|
|
has("prsidente", "presidente");
|
|
has("provavel", "provável");
|
|
has("trbalho", "trabalho");
|
|
has("nãoé", "não é");
|
|
has("paraela", "para ela");
|
|
has("brasília", "Brasília");
|
|
has("Coraçao", "Coração");
|
|
}
|
|
|
|
#[test]
|
|
fn the_likeliest_fix_comes_first() {
|
|
crate::pt::test_dict::install();
|
|
assert_eq!(suggest("coraçao", 3).first().map(String::as_str), Some("coração"));
|
|
assert_eq!(
|
|
suggest("saude", 3).first().map(String::as_str),
|
|
Some("saúde"),
|
|
"an accent at the first difference beats one further right (saudê is a word too)"
|
|
);
|
|
assert_eq!(suggest("seje", 3).first().map(String::as_str), Some("seja"));
|
|
}
|
|
|
|
#[test]
|
|
fn edge_cases_stay_empty_or_bounded() {
|
|
crate::pt::test_dict::install();
|
|
assert!(suggest("", 5).is_empty());
|
|
assert!(suggest("bem", 0).is_empty());
|
|
assert!(suggest(&"a".repeat(41), 5).is_empty());
|
|
assert!(suggest("duas palavras", 5).is_empty());
|
|
assert!(suggest_in(&crate::affix::AffixDict::default(), "coraçao", 5).is_empty());
|
|
assert!(suggest("coração", 5).len() <= 5);
|
|
assert!(!suggest("coração", 5).contains(&"coração".to_string()));
|
|
}
|
|
|
|
#[test]
|
|
fn distance_weighs_accents_half() {
|
|
let map = vec![vec!['a', 'á', 'à', 'â', 'ã'], vec!['c', 'ç']];
|
|
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
|
|
let c = |s: &str| s.chars().collect::<Vec<_>>();
|
|
assert_eq!(weighted_distance(&c("coraçao"), &c("coração"), &related, 9), 1);
|
|
assert_eq!(weighted_distance(&c("coracao"), &c("coração"), &related, 9), 2, "ç and ã are two accent slips");
|
|
assert_eq!(weighted_distance(&c("coro"), &c("coro"), &related, 9), 0);
|
|
assert_eq!(weighted_distance(&c("bem"), &c("bom"), &related, 9), 2);
|
|
assert_eq!(weighted_distance(&c("bem"), &c("bemte"), &related, 9), 4);
|
|
assert_eq!(weighted_distance(&c("bem"), &c("ver"), &related, 2), 3);
|
|
assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4);
|
|
}
|
|
|
|
#[test]
|
|
fn suggestions_are_fast_enough() {
|
|
crate::pt::test_dict::install();
|
|
// Warm the dictionary, then time a worst case (a long word nothing near): the scan round.
|
|
let _ = suggest("bem", 1);
|
|
let t = std::time::Instant::now();
|
|
for w in ["xqzvweportuguespalavra", "prsidentes", "paralelepípedo"] {
|
|
let _ = suggest(w, 6);
|
|
}
|
|
let ms = t.elapsed().as_secs_f64() * 1000.0;
|
|
eprintln!("3 suggestion calls: {ms:.0} ms");
|
|
assert!(ms < 20_000.0, "{ms} ms");
|
|
}
|
|
}
|