iced_highlighter/
lib.rs

1//! A syntax highlighter for iced.
2use iced_core as core;
3
4use crate::core::Code;
5use crate::core::text;
6
7use std::ops::Range;
8use std::sync::LazyLock;
9
10use syntect::parsing;
11use two_face::re_exports::syntect;
12
13static SYNTAXES: LazyLock<parsing::SyntaxSet> = LazyLock::new(two_face::syntax::extra_no_newlines);
14
15const LINES_PER_SNAPSHOT: usize = 50;
16
17/// A syntax parser.
18#[derive(Debug)]
19pub struct Parser {
20    syntax: &'static parsing::SyntaxReference,
21    caches: Vec<(parsing::ParseState, parsing::ScopeStack)>,
22    current_line: usize,
23}
24
25/// An iterator over the highlighted regions of a line.
26///
27/// Each item is a character range within the line, paired with
28/// the [`Code`] of the region.
29pub type CodeIterator<'a> = Box<dyn Iterator<Item = (Range<usize>, Code)> + 'a>;
30
31impl Parser {
32    /// Creates a new [`Parser`] with the given [`Settings`].
33    pub fn new(settings: &Settings) -> Self {
34        let syntax = SYNTAXES
35            .find_syntax_by_token(&settings.token)
36            .unwrap_or_else(|| SYNTAXES.find_syntax_plain_text());
37
38        let parser = parsing::ParseState::new(syntax);
39        let stack = parsing::ScopeStack::new();
40
41        Parser {
42            syntax,
43            caches: vec![(parser, stack)],
44            current_line: 0,
45        }
46    }
47
48    /// Updates the parser with the given [`Settings`],
49    /// restarting it from the first line.
50    pub fn update(&mut self, new_settings: &Settings) {
51        self.syntax = SYNTAXES
52            .find_syntax_by_token(&new_settings.token)
53            .unwrap_or_else(|| SYNTAXES.find_syntax_plain_text());
54
55        // Restart the parser
56        self.change_line(0);
57    }
58
59    /// Changes the line the parser is currently on.
60    pub fn change_line(&mut self, line: usize) {
61        let snapshot = line / LINES_PER_SNAPSHOT;
62
63        if snapshot <= self.caches.len() {
64            self.caches.truncate(snapshot);
65            self.current_line = snapshot * LINES_PER_SNAPSHOT;
66        } else {
67            self.caches.truncate(1);
68            self.current_line = 0;
69        }
70
71        let (parser, stack) = self.caches.last().cloned().unwrap_or_else(|| {
72            (
73                parsing::ParseState::new(self.syntax),
74                parsing::ScopeStack::new(),
75            )
76        });
77
78        self.caches.push((parser, stack));
79    }
80
81    /// Highlights the given line, returning a [`CodeIterator`].
82    pub fn parse_line(&mut self, line: &str) -> CodeIterator<'_> {
83        if self.current_line / LINES_PER_SNAPSHOT >= self.caches.len() {
84            let (parser, stack) = self.caches.last().expect("Caches must not be empty");
85
86            self.caches.push((parser.clone(), stack.clone()));
87        }
88
89        self.current_line += 1;
90
91        let (parser, stack) = self.caches.last_mut().expect("Caches must not be empty");
92        let ops = parser.parse_line(line, &SYNTAXES).unwrap_or_default();
93
94        Box::new(code_iterator(ops, line, stack))
95    }
96
97    /// Returns the line the parser is currently on.
98    pub fn current_line(&self) -> usize {
99        self.current_line
100    }
101}
102
103impl text::Parser for Parser {
104    type Settings = Settings;
105    type Output = Code;
106    type Iterator<'a> = CodeIterator<'a>;
107
108    fn new(settings: &Self::Settings) -> Self {
109        Self::new(settings)
110    }
111
112    fn update(&mut self, new_settings: &Self::Settings) {
113        self.update(new_settings);
114    }
115
116    fn change_line(&mut self, line: usize) {
117        self.change_line(line);
118    }
119
120    fn parse_line(&mut self, line: &str) -> Self::Iterator<'_> {
121        self.parse_line(line)
122    }
123
124    fn current_line(&self) -> usize {
125        self.current_line()
126    }
127}
128
129fn code_iterator<'a>(
130    ops: Vec<(usize, parsing::ScopeStackOp)>,
131    line: &str,
132    stack: &'a mut parsing::ScopeStack,
133) -> impl Iterator<Item = (Range<usize>, Code)> + 'a {
134    ScopeRangeIterator {
135        ops,
136        line_length: line.len(),
137        index: 0,
138        last_str_index: 0,
139    }
140    .filter_map(move |(range, scope)| {
141        let _ = stack.apply(&scope);
142
143        if range.is_empty() {
144            None
145        } else {
146            Some((range, scope_from_stack(&stack.scopes)))
147        }
148    })
149}
150
151/// A streaming syntax parser.
152///
153/// It can efficiently highlight an immutable stream of tokens.
154#[derive(Debug)]
155pub struct Stream {
156    syntax: &'static parsing::SyntaxReference,
157    commit: (parsing::ParseState, parsing::ScopeStack),
158    state: parsing::ParseState,
159    stack: parsing::ScopeStack,
160}
161
162impl Stream {
163    /// Creates a new [`Stream`] parser.
164    pub fn new(settings: &Settings) -> Self {
165        let syntax = SYNTAXES
166            .find_syntax_by_token(&settings.token)
167            .unwrap_or_else(|| SYNTAXES.find_syntax_plain_text());
168
169        let state = parsing::ParseState::new(syntax);
170        let stack = parsing::ScopeStack::new();
171
172        Self {
173            syntax,
174            commit: (state.clone(), stack.clone()),
175            state,
176            stack,
177        }
178    }
179
180    /// Highlights the given line from the last commit.
181    pub fn parse_line(&mut self, line: &str) -> impl Iterator<Item = (Range<usize>, Code)> + '_ {
182        self.state = self.commit.0.clone();
183        self.stack = self.commit.1.clone();
184
185        let ops = self.state.parse_line(line, &SYNTAXES).unwrap_or_default();
186        code_iterator(ops, line, &mut self.stack)
187    }
188
189    /// Commits the last highlighted line.
190    pub fn commit(&mut self) {
191        self.commit = (self.state.clone(), self.stack.clone());
192    }
193
194    /// Resets the [`Stream`] parser.
195    pub fn reset(&mut self) {
196        self.state = parsing::ParseState::new(self.syntax);
197        self.stack = parsing::ScopeStack::new();
198        self.commit = (self.state.clone(), self.stack.clone());
199    }
200}
201
202/// The settings of a [`Parser`].
203#[derive(Debug, Clone, PartialEq)]
204pub struct Settings {
205    /// The extension of the file or the name of the language to highlight.
206    ///
207    /// The [`Parser`] will use the token to automatically determine
208    /// the grammar to use for highlighting.
209    pub token: String,
210}
211
212/// The scope families and their classes.
213///
214/// The families are listed most specific first, so that e.g.
215/// `entity.name.function` wins over `entity.name`.
216static FAMILIES: LazyLock<Vec<(parsing::Scope, Code)>> = LazyLock::new(|| {
217    [
218        ("meta.path", Code::Path),
219        ("invalid", Code::Invalid),
220        ("constant", Code::Constant),
221        ("string", Code::String),
222        ("comment", Code::Comment),
223        ("keyword", Code::Keyword),
224        ("storage.type.", Code::Keyword),
225        ("storage.type", Code::Type),
226        ("storage", Code::Keyword),
227        ("entity.name.function", Code::Function),
228        ("entity.name", Code::Type),
229        ("entity.other.inherited-class", Code::Type),
230        ("support", Code::Support),
231        ("variable.function", Code::Function),
232        ("variable", Code::Variable),
233        ("punctuation", Code::Punctuation),
234    ]
235    .into_iter()
236    .map(|(name, class)| {
237        (
238            parsing::Scope::new(name).expect("scope family is valid"),
239            class,
240        )
241    })
242    .collect()
243});
244
245/// Classifies the scope stack of a highlighted region.
246///
247/// The stack is walked from the most specific scope (last) to the
248/// least specific (first); the first scope that matches a family
249/// determines the class. If no scope matches, the region is
250/// classified as [`Code::Other`].
251fn scope_from_stack(stack: &[parsing::Scope]) -> Code {
252    for scope in stack.iter().rev() {
253        for (family, class) in FAMILIES.iter() {
254            if family.is_prefix_of(*scope) {
255                return *class;
256            }
257        }
258    }
259
260    Code::Other
261}
262
263struct ScopeRangeIterator {
264    ops: Vec<(usize, parsing::ScopeStackOp)>,
265    line_length: usize,
266    index: usize,
267    last_str_index: usize,
268}
269
270impl Iterator for ScopeRangeIterator {
271    type Item = (std::ops::Range<usize>, parsing::ScopeStackOp);
272
273    fn next(&mut self) -> Option<Self::Item> {
274        if self.index > self.ops.len() {
275            return None;
276        }
277
278        let next_str_i = if self.index == self.ops.len() {
279            self.line_length
280        } else {
281            self.ops[self.index].0
282        };
283
284        let range = self.last_str_index..next_str_i;
285        self.last_str_index = next_str_i;
286
287        let op = if self.index == 0 {
288            parsing::ScopeStackOp::Noop
289        } else {
290            self.ops[self.index - 1].1.clone()
291        };
292
293        self.index += 1;
294        Some((range, op))
295    }
296}
297
298#[cfg(test)]
299mod tests {
300    use super::*;
301
302    /// Builds a scope stack from dotted scope names.
303    fn stack(names: &[&str]) -> Vec<parsing::Scope> {
304        names
305            .iter()
306            .map(|name| parsing::Scope::new(name).unwrap())
307            .collect()
308    }
309
310    #[test]
311    fn scopes_are_classified_by_family() {
312        assert_eq!(
313            scope_from_stack(&stack(&["source.rust", "comment.line"])),
314            Code::Comment
315        );
316        assert_eq!(
317            scope_from_stack(&stack(&["source.rust", "comment.block"])),
318            Code::Comment
319        );
320        assert_eq!(
321            scope_from_stack(&stack(&["source.rust", "string.quoted.double"])),
322            Code::String
323        );
324        assert_eq!(
325            scope_from_stack(&stack(&["source.rust", "keyword.control"])),
326            Code::Keyword
327        );
328        assert_eq!(
329            scope_from_stack(&stack(&["source.rust", "keyword.operator"])),
330            Code::Keyword
331        );
332        assert_eq!(
333            scope_from_stack(&stack(&["source.rust", "storage.type"])),
334            Code::Keyword
335        );
336        assert_eq!(
337            scope_from_stack(&stack(&["source.rust", "constant.numeric"])),
338            Code::Constant
339        );
340        assert_eq!(
341            scope_from_stack(&stack(&["source.rust", "entity.name.function"])),
342            Code::Function
343        );
344        assert_eq!(
345            scope_from_stack(&stack(&["source.rust", "entity.name.type"])),
346            Code::Type
347        );
348        assert_eq!(
349            scope_from_stack(&stack(&["source.rust", "entity.other.inherited-class"])),
350            Code::Type
351        );
352        assert_eq!(
353            scope_from_stack(&stack(&["source.rust", "variable.parameter"])),
354            Code::Variable
355        );
356        assert_eq!(
357            scope_from_stack(&stack(&["source.rust", "support.function.builtin"])),
358            Code::Support
359        );
360        assert_eq!(
361            scope_from_stack(&stack(&["source.rust", "punctuation.definition"])),
362            Code::Punctuation
363        );
364        assert_eq!(
365            scope_from_stack(&stack(&["source.rust", "invalid.illegal"])),
366            Code::Invalid
367        );
368    }
369
370    #[test]
371    fn the_most_specific_scope_wins() {
372        // An escape sequence inside a string is a constant, not a string.
373        let names = [
374            "source.rust",
375            "string.quoted.double",
376            "constant.character.escape",
377        ];
378
379        assert_eq!(scope_from_stack(&stack(&names)), Code::Constant);
380
381        // A comment marker is punctuation, even inside a comment.
382        let names = [
383            "source.rust",
384            "comment.line",
385            "punctuation.definition.comment",
386        ];
387
388        assert_eq!(scope_from_stack(&stack(&names)), Code::Punctuation);
389    }
390
391    #[test]
392    fn the_walk_continues_to_shallower_scopes() {
393        // The leaf scope is unmatched, so the classification falls back
394        // to the next scope in the stack.
395        let names = ["source.rust", "string.quoted.double", "meta.embedded"];
396
397        assert_eq!(scope_from_stack(&stack(&names)), Code::String);
398    }
399
400    #[test]
401    fn unmatched_scopes_are_other() {
402        assert_eq!(scope_from_stack(&[]), Code::Other);
403        assert_eq!(
404            scope_from_stack(&stack(&["source.rust", "meta.function"])),
405            Code::Other
406        );
407
408        // A family must match whole scope atoms: `stringify` is not a
409        // `string`.
410        assert_eq!(
411            scope_from_stack(&stack(&["source.rust", "stringify.call"])),
412            Code::Other
413        );
414    }
415}
iced_highlighter/lib.rs

iced_highlighter/
lib.rs
