Move highlighting logic from JS to Rust

Continue migrating JS functionality

Cleanup

Fix compile error

Clean up the diff

Set toggle font to sans-serif
This commit is contained in:
Will Crichton 2021-08-26 14:43:12 -07:00
parent eea8f0a39a
commit 55bb51786e
10 changed files with 188 additions and 186 deletions

View file

@ -45,7 +45,7 @@ crate struct TestOptions {
crate attrs: Vec<String>,
}
crate fn make_rustc_config(options: &Options) -> interface::Config {
crate fn run(options: Options) -> Result<(), ErrorReported> {
let input = config::Input::File(options.input.clone());
let invalid_codeblock_attributes_name = crate::lint::INVALID_CODEBLOCK_ATTRIBUTES.name;
@ -87,7 +87,7 @@ crate fn make_rustc_config(options: &Options) -> interface::Config {
let mut cfgs = options.cfgs.clone();
cfgs.push("doc".to_owned());
cfgs.push("doctest".to_owned());
interface::Config {
let config = interface::Config {
opts: sessopts,
crate_cfg: interface::parse_cfgspecs(cfgs),
input,
@ -103,11 +103,7 @@ crate fn make_rustc_config(options: &Options) -> interface::Config {
override_queries: None,
make_codegen_backend: None,
registry: rustc_driver::diagnostics_registry(),
}
}
crate fn run(options: Options) -> Result<(), ErrorReported> {
let config = make_rustc_config(&options);
};
let test_args = options.test_args.clone();
let display_doctest_warnings = options.display_doctest_warnings;

View file

@ -12,6 +12,7 @@ use crate::html::render::Context;
use std::collections::VecDeque;
use std::fmt::{Display, Write};
use rustc_data_structures::fx::FxHashMap;
use rustc_lexer::{LiteralKind, TokenKind};
use rustc_span::edition::Edition;
use rustc_span::symbol::Symbol;
@ -30,6 +31,8 @@ crate struct ContextInfo<'a, 'b, 'c> {
crate root_path: &'c str,
}
crate type DecorationInfo = FxHashMap<&'static str, Vec<(u32, u32)>>;
/// Highlights `src`, returning the HTML output.
crate fn render_with_highlighting(
src: &str,
@ -40,6 +43,7 @@ crate fn render_with_highlighting(
edition: Edition,
extra_content: Option<Buffer>,
context_info: Option<ContextInfo<'_, '_, '_>>,
decoration_info: Option<DecorationInfo>,
) {
debug!("highlighting: ================\n{}\n==============", src);
if let Some((edition_info, class)) = tooltip {
@ -56,7 +60,7 @@ crate fn render_with_highlighting(
}
write_header(out, class, extra_content);
write_code(out, &src, edition, context_info);
write_code(out, &src, edition, context_info, decoration_info);
write_footer(out, playground_button);
}
@ -89,17 +93,23 @@ fn write_code(
src: &str,
edition: Edition,
context_info: Option<ContextInfo<'_, '_, '_>>,
decoration_info: Option<DecorationInfo>,
) {
// This replace allows to fix how the code source with DOS backline characters is displayed.
let src = src.replace("\r\n", "\n");
Classifier::new(&src, edition, context_info.as_ref().map(|c| c.file_span).unwrap_or(DUMMY_SP))
.highlight(&mut |highlight| {
match highlight {
Highlight::Token { text, class } => string(out, Escape(text), class, &context_info),
Highlight::EnterSpan { class } => enter_span(out, class),
Highlight::ExitSpan => exit_span(out),
};
});
Classifier::new(
&src,
edition,
context_info.as_ref().map(|c| c.file_span).unwrap_or(DUMMY_SP),
decoration_info,
)
.highlight(&mut |highlight| {
match highlight {
Highlight::Token { text, class } => string(out, Escape(text), class, &context_info),
Highlight::EnterSpan { class } => enter_span(out, class),
Highlight::ExitSpan => exit_span(out),
};
});
}
fn write_footer(out: &mut Buffer, playground_button: Option<&str>) {
@ -127,6 +137,7 @@ enum Class {
PreludeTy,
PreludeVal,
QuestionMark,
Decoration(&'static str),
}
impl Class {
@ -150,6 +161,7 @@ impl Class {
Class::PreludeTy => "prelude-ty",
Class::PreludeVal => "prelude-val",
Class::QuestionMark => "question-mark",
Class::Decoration(kind) => kind,
}
}
@ -244,7 +256,28 @@ impl Iterator for PeekIter<'a> {
type Item = (TokenKind, &'a str);
fn next(&mut self) -> Option<Self::Item> {
self.peek_pos = 0;
if let Some(first) = self.stored.pop_front() { Some(first) } else { self.iter.next() }
if let Some(first) = self.stored.pop_front() {
Some(first)
} else {
self.iter.next()
}
}
}
/// Custom spans inserted into the source. Eg --scrape-examples uses this to highlight function calls
struct Decorations {
starts: Vec<(u32, &'static str)>,
ends: Vec<u32>,
}
impl Decorations {
fn new(info: DecorationInfo) -> Self {
let (starts, ends) = info
.into_iter()
.map(|(kind, ranges)| ranges.into_iter().map(move |(lo, hi)| ((lo, kind), hi)))
.flatten()
.unzip();
Decorations { starts, ends }
}
}
@ -259,12 +292,18 @@ struct Classifier<'a> {
byte_pos: u32,
file_span: Span,
src: &'a str,
decorations: Option<Decorations>,
}
impl<'a> Classifier<'a> {
/// Takes as argument the source code to HTML-ify, the rust edition to use and the source code
/// file span which will be used later on by the `span_correspondance_map`.
fn new(src: &str, edition: Edition, file_span: Span) -> Classifier<'_> {
fn new(
src: &str,
edition: Edition,
file_span: Span,
decoration_info: Option<DecorationInfo>,
) -> Classifier<'_> {
let tokens = PeekIter::new(TokenIter { src });
Classifier {
tokens,
@ -275,6 +314,7 @@ impl<'a> Classifier<'a> {
byte_pos: 0,
file_span,
src,
decorations,
}
}
@ -356,6 +396,19 @@ impl<'a> Classifier<'a> {
/// token is used.
fn highlight(mut self, sink: &mut dyn FnMut(Highlight<'a>)) {
loop {
if let Some(decs) = self.decorations.as_mut() {
let byte_pos = self.byte_pos;
let n_starts = decs.starts.iter().filter(|(i, _)| byte_pos >= *i).count();
for (_, kind) in decs.starts.drain(0..n_starts) {
sink(Highlight::EnterSpan { class: Class::Decoration(kind) });
}
let n_ends = decs.ends.iter().filter(|i| byte_pos >= **i).count();
for _ in decs.ends.drain(0..n_ends) {
sink(Highlight::ExitSpan);
}
}
if self
.tokens
.peek()

View file

@ -22,7 +22,7 @@ fn test_html_highlighting() {
let src = include_str!("fixtures/sample.rs");
let html = {
let mut out = Buffer::new();
write_code(&mut out, src, Edition::Edition2018, None);
write_code(&mut out, src, Edition::Edition2018, None, None);
format!("{}<pre><code>{}</code></pre>\n", STYLE, out.into_inner())
};
expect_file!["fixtures/sample.html"].assert_eq(&html);
@ -36,7 +36,7 @@ fn test_dos_backline() {
println!(\"foo\");\r\n\
}\r\n";
let mut html = Buffer::new();
write_code(&mut html, src, Edition::Edition2018, None);
write_code(&mut html, src, Edition::Edition2018, None, None);
expect_file!["fixtures/dos_line.html"].assert_eq(&html.into_inner());
});
}
@ -50,7 +50,7 @@ let x = super::b::foo;
let y = Self::whatever;";
let mut html = Buffer::new();
write_code(&mut html, src, Edition::Edition2018, None);
write_code(&mut html, src, Edition::Edition2018, None, None);
expect_file!["fixtures/highlight.html"].assert_eq(&html.into_inner());
});
}

View file

@ -360,6 +360,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'_, 'a, I> {
edition,
None,
None,
None,
);
Some(Event::Html(s.into_inner().into()))
}

View file

@ -46,7 +46,7 @@ use std::string::ToString;
use rustc_ast_pretty::pprust;
use rustc_attr::{ConstStability, Deprecation, StabilityLevel};
use rustc_data_structures::fx::FxHashSet;
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_hir as hir;
use rustc_hir::def::CtorKind;
use rustc_hir::def_id::DefId;
@ -2496,23 +2496,28 @@ fn render_call_locations(
// To reduce file sizes, we only want to embed the source code needed to understand the example, not
// the entire file. So we find the smallest byte range that covers all items enclosing examples.
assert!(call_data.locations.len() > 0);
let min_loc =
call_data.locations.iter().min_by_key(|loc| loc.enclosing_item_span.0).unwrap();
let min_byte = min_loc.enclosing_item_span.0;
let min_line = min_loc.enclosing_item_lines.0;
call_data.locations.iter().min_by_key(|loc| loc.enclosing_item.byte_span.0).unwrap();
let min_byte = min_loc.enclosing_item.byte_span.0;
let min_line = min_loc.enclosing_item.line_span.0;
let max_byte =
call_data.locations.iter().map(|loc| loc.enclosing_item_span.1).max().unwrap();
call_data.locations.iter().map(|loc| loc.enclosing_item.byte_span.1).max().unwrap();
// The output code is limited to that byte range.
let contents_subset = &contents[min_byte..max_byte];
let contents_subset = &contents[(min_byte as usize)..(max_byte as usize)];
// The call locations need to be updated to reflect that the size of the program has changed.
// Specifically, the ranges are all subtracted by `min_byte` since that's the new zero point.
let locations = call_data
let (byte_ranges, line_ranges): (Vec<_>, Vec<_>) = call_data
.locations
.iter()
.map(|loc| (loc.call_span.0 - min_byte, loc.call_span.1 - min_byte))
.collect::<Vec<_>>();
.map(|loc| {
let (byte_lo, byte_hi) = loc.call_expr.byte_span;
let (line_lo, line_hi) = loc.call_expr.line_span;
((byte_lo - min_byte, byte_hi - min_byte), (line_lo - min_line, line_hi - min_line))
})
.unzip();
let edition = cx.shared.edition();
write!(
@ -2524,7 +2529,7 @@ fn render_call_locations(
// The code and locations are encoded as data attributes, so they can be read
// later by the JS for interactions.
code = contents_subset.replace("\"", "&quot;"),
locations = serde_json::to_string(&locations).unwrap(),
locations = serde_json::to_string(&line_ranges).unwrap(),
);
write!(w, r#"<span class="prev">&pr;</span> <span class="next">&sc;</span>"#);
write!(w, r#"<span class="expand">&varr;</span>"#);
@ -2532,7 +2537,18 @@ fn render_call_locations(
// FIXME(wcrichto): where should file_span and root_path come from?
let file_span = rustc_span::DUMMY_SP;
let root_path = "".to_string();
sources::print_src(w, contents_subset, edition, file_span, cx, &root_path, Some(min_line));
let mut decoration_info = FxHashMap::default();
decoration_info.insert("highlight", byte_ranges);
sources::print_src(
w,
contents_subset,
edition,
file_span,
cx,
&root_path,
Some(min_line),
Some(decoration_info),
);
write!(w, "</div></div>");
};
@ -2542,7 +2558,8 @@ fn render_call_locations(
// understand at a glance.
let ordered_locations = {
let sort_criterion = |(_, call_data): &(_, &CallData)| {
let (lo, hi) = call_data.locations[0].enclosing_item_span;
// Use the first location because that's what the user will see initially
let (lo, hi) = call_data.locations[0].enclosing_item.byte_span;
hi - lo
};

View file

@ -1117,6 +1117,7 @@ fn item_macro(w: &mut Buffer, cx: &Context<'_>, it: &clean::Item, t: &clean::Mac
it.span(cx.tcx()).inner().edition(),
None,
None,
None,
);
});
document(w, cx, it, None, HeadingOffset::H2)

View file

@ -212,6 +212,7 @@ impl SourceCollector<'_, 'tcx> {
&self.cx,
&root_path,
None,
None,
)
},
&self.cx.shared.style_files,
@ -259,6 +260,7 @@ crate fn print_src(
context: &Context<'_>,
root_path: &str,
offset: Option<usize>,
decoration_info: Option<highlight::DecorationInfo>,
) {
let lines = s.lines().count();
let mut line_numbers = Buffer::empty_from(buf);
@ -283,5 +285,6 @@ crate fn print_src(
edition,
Some(line_numbers),
Some(highlight::ContextInfo { context, file_span, root_path }),
decoration_info,
);
}

View file

@ -137,7 +137,7 @@ h1.fqn {
margin-top: 0;
/* workaround to keep flex from breaking below 700 px width due to the float: right on the nav
above the h1 */
above the h1 */
padding-left: 1px;
}
h1.fqn > .in-band > a:hover {
@ -974,7 +974,7 @@ body.blur > :not(#help) {
text-shadow:
1px 0 0 black,
-1px 0 0 black,
0 1px 0 black,
0 1px 0 black,
0 -1px 0 black;
}
@ -1214,8 +1214,8 @@ a.test-arrow:hover{
.notable-traits-tooltip::after {
/* The margin on the tooltip does not capture hover events,
this extends the area of hover enough so that mouse hover is not
lost when moving the mouse to the tooltip */
this extends the area of hover enough so that mouse hover is not
lost when moving the mouse to the tooltip */
content: "\00a0\00a0\00a0";
}
@ -1715,7 +1715,7 @@ details.undocumented[open] > summary::before {
}
/* We do NOT hide this element so that alternative device readers still have this information
available. */
available. */
.sidebar-elems {
position: fixed;
z-index: 1;
@ -1971,7 +1971,8 @@ details.undocumented[open] > summary::before {
}
}
/* This part is for the new "examples" components */
/* Begin: styles for --scrape-examples feature */
.scraped-example-title {
font-family: 'Fira Sans';
@ -2063,16 +2064,17 @@ details.undocumented[open] > summary::before {
overflow-y: hidden;
}
.scraped-example .line-numbers span.highlight {
background: #f6fdb0;
.scraped-example .example-wrap .rust span.highlight {
background: #fcffd6;
}
.scraped-example .example-wrap .rust span.highlight {
.scraped-example .example-wrap .rust span.highlight.focus {
background: #f6fdb0;
}
.more-examples-toggle summary {
color: #999;
font-family: 'Fira Sans';
}
.more-scraped-examples {
@ -2115,3 +2117,5 @@ h1 + .scraped-example {
.example-links ul {
margin-bottom: 0;
}
/* End: styles for --scrape-examples feature */

View file

@ -980,154 +980,55 @@ function hideThemeButtonState() {
window.addEventListener("hashchange", onHashChange);
searchState.setup();
/////// EXAMPLE ANALYZER
// Merge the full set of [from, to] offsets into a minimal set of non-overlapping
// [from, to] offsets.
// NB: This is such a archetypal software engineering interview question that
// I can't believe I actually had to write it. Yes, it's O(N) in the input length --
// but it does assume a sorted input!
function distinctRegions(locs) {
var start = -1;
var end = -1;
var output = [];
for (var i = 0; i < locs.length; i++) {
var loc = locs[i];
if (loc[0] > end) {
if (end > 0) {
output.push([start, end]);
}
start = loc[0];
end = loc[1];
} else {
end = Math.max(end, loc[1]);
}
}
if (end > 0) {
output.push([start, end]);
}
return output;
}
function convertLocsStartsToLineOffsets(code, locs) {
locs = distinctRegions(locs.slice(0).sort(function (a, b) {
return a[0] === b[0] ? a[1] - b[1] : a[0] - b[0];
})); // sort by start; use end if start is equal.
var codeLines = code.split("\n");
var lineIndex = 0;
var totalOffset = 0;
var output = [];
while (locs.length > 0 && lineIndex < codeLines.length) {
// +1 here and later is due to omitted \n
var lineLength = codeLines[lineIndex].length + 1;
while (locs.length > 0 && totalOffset + lineLength > locs[0][0]) {
var endIndex = lineIndex;
var charsRemaining = locs[0][1] - totalOffset;
while (endIndex < codeLines.length &&
charsRemaining > codeLines[endIndex].length + 1)
{
charsRemaining -= codeLines[endIndex].length + 1;
endIndex += 1;
}
output.push({
from: [lineIndex, locs[0][0] - totalOffset],
to: [endIndex, charsRemaining]
});
locs.shift();
}
lineIndex++;
totalOffset += lineLength;
}
return output;
}
// inserts str into html, *but* calculates idx by eliding anything in html that's not in raw.
// ideally this would work by walking the element tree...but this is good enough for now.
function insertStrAtRawIndex(raw, html, idx, str) {
if (idx > raw.length) {
return html;
}
if (idx == raw.length) {
return html + str;
}
var rawIdx = 0;
var htmlIdx = 0;
while (rawIdx < idx && rawIdx < raw.length) {
while (raw[rawIdx] !== html[htmlIdx] && htmlIdx < html.length) {
htmlIdx++;
}
rawIdx++;
htmlIdx++;
}
return html.substring(0, htmlIdx) + str + html.substr(htmlIdx);
}
/////// --scrape-examples interactions
// Scroll code block to put the given code location in the middle of the viewer
function scrollToLoc(elt, loc) {
var wrapper = elt.querySelector(".code-wrapper");
var halfHeight = wrapper.offsetHeight / 2;
var lines = elt.querySelector('.line-numbers');
var offsetMid = (lines.children[loc.from[0]].offsetTop
+ lines.children[loc.to[0]].offsetTop) / 2;
var offsetMid = (lines.children[loc[0]].offsetTop
+ lines.children[loc[1]].offsetTop) / 2;
var scrollOffset = offsetMid - halfHeight;
lines.scrollTo(0, scrollOffset);
elt.querySelector(".rust").scrollTo(0, scrollOffset);
}
function updateScrapedExample(example) {
var code = example.attributes.getNamedItem("data-code").textContent;
var codeLines = code.split("\n");
var locs = JSON.parse(example.attributes.getNamedItem("data-locs").textContent);
locs = convertLocsStartsToLineOffsets(code, locs);
// Add call-site highlights to code listings
var litParent = example.querySelector('.example-wrap pre.rust');
var litHtml = litParent.innerHTML.split("\n");
onEach(locs, function (loc) {
for (var i = loc.from[0]; i < loc.to[0] + 1; i++) {
addClass(example.querySelector('.line-numbers').children[i], "highlight");
}
litHtml[loc.to[0]] = insertStrAtRawIndex(
codeLines[loc.to[0]],
litHtml[loc.to[0]],
loc.to[1],
"</span>");
litHtml[loc.from[0]] = insertStrAtRawIndex(
codeLines[loc.from[0]],
litHtml[loc.from[0]],
loc.from[1],
'<span class="highlight" data-loc="' +
JSON.stringify(loc).replace(/"/g, "&quot;") +
'">');
}, true); // do this backwards to avoid shifting later offsets
litParent.innerHTML = litHtml.join('\n');
// Toggle through list of examples in a given file
var locIndex = 0;
var highlights = example.querySelectorAll('.highlight');
addClass(highlights[0], 'focus');
if (locs.length > 1) {
// Toggle through list of examples in a given file
var onChangeLoc = function(f) {
removeClass(highlights[locIndex], 'focus');
f();
scrollToLoc(example, locs[locIndex]);
addClass(highlights[locIndex], 'focus');
};
example.querySelector('.prev')
.addEventListener('click', function () {
locIndex = (locIndex - 1 + locs.length) % locs.length;
scrollToLoc(example, locs[locIndex]);
.addEventListener('click', function() {
onChangeLoc(function() {
locIndex = (locIndex - 1 + locs.length) % locs.length;
});
});
example.querySelector('.next')
.addEventListener('click', function () {
locIndex = (locIndex + 1) % locs.length;
scrollToLoc(example, locs[locIndex]);
.addEventListener('click', function() {
onChangeLoc(function() { locIndex = (locIndex + 1) % locs.length; });
});
} else {
// Remove buttons if there's only one example in the file
example.querySelector('.prev').remove();
example.querySelector('.next').remove();
}
let codeEl = example.querySelector('.rust');
let expandButton = example.querySelector('.expand');
if (codeEl.scrollHeight == codeEl.clientHeight) {
addClass(example, 'expanded');
expandButton.remove();
} else {
// Show full code on expansion
var codeEl = example.querySelector('.rust');
var codeOverflows = codeEl.scrollHeight > codeEl.clientHeight;
var expandButton = example.querySelector('.expand');
if (codeOverflows) {
// If file is larger than default height, give option to expand the viewer
expandButton.addEventListener('click', function () {
if (hasClass(example, "expanded")) {
removeClass(example, "expanded");
@ -1136,6 +1037,10 @@ function hideThemeButtonState() {
addClass(example, "expanded");
}
});
} else {
// Otherwise remove expansion buttons
addClass(example, 'expanded');
expandButton.remove();
}
// Start with the first example in view
@ -1146,6 +1051,8 @@ function hideThemeButtonState() {
var firstExamples = document.querySelectorAll('.scraped-example-list > .scraped-example');
onEach(firstExamples, updateScrapedExample);
onEach(document.querySelectorAll('.more-examples-toggle'), function(toggle) {
// Allow users to click the left border of the <details> section to close it,
// since the section can be large and finding the [+] button is annoying.
toggle.querySelector('.toggle-line').addEventListener('click', function() {
toggle.open = false;
});

View file

@ -1,5 +1,4 @@
//! This module analyzes crates to find examples of uses for items in the
//! current crate being documented.
//! This module analyzes crates to find call sites that can serve as examples in the documentation.
use crate::clean;
use crate::config;
@ -11,20 +10,55 @@ use rustc_data_structures::fx::FxHashMap;
use rustc_hir::{
self as hir,
intravisit::{self, Visitor},
HirId,
};
use rustc_interface::interface;
use rustc_middle::hir::map::Map;
use rustc_middle::ty::{self, TyCtxt};
use rustc_span::{def_id::DefId, FileName};
use rustc_span::{def_id::DefId, BytePos, FileName, SourceFile};
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::PathBuf;
#[derive(Serialize, Deserialize, Debug, Clone)]
crate struct SyntaxRange {
crate byte_span: (u32, u32),
crate line_span: (usize, usize),
}
impl SyntaxRange {
fn new(span: rustc_span::Span, file: &SourceFile) -> Self {
let get_pos = |bytepos: BytePos| file.original_relative_byte_pos(bytepos).0;
let get_line = |bytepos: BytePos| file.lookup_line(bytepos).unwrap();
SyntaxRange {
byte_span: (get_pos(span.lo()), get_pos(span.hi())),
line_span: (get_line(span.lo()), get_line(span.hi())),
}
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
crate struct CallLocation {
crate call_span: (usize, usize),
crate enclosing_item_span: (usize, usize),
crate enclosing_item_lines: (usize, usize),
crate call_expr: SyntaxRange,
crate enclosing_item: SyntaxRange,
}
impl CallLocation {
fn new(
tcx: TyCtxt<'_>,
expr_span: rustc_span::Span,
expr_id: HirId,
source_file: &rustc_span::SourceFile,
) -> Self {
let enclosing_item_span = tcx.hir().span_with_body(tcx.hir().get_parent_item(expr_id));
assert!(enclosing_item_span.contains(expr_span));
CallLocation {
call_expr: SyntaxRange::new(expr_span, source_file),
enclosing_item: SyntaxRange::new(enclosing_item_span, source_file),
}
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
@ -96,24 +130,10 @@ where
_ => None,
};
let get_pos =
|bytepos: rustc_span::BytePos| file.original_relative_byte_pos(bytepos).0 as usize;
let get_range = |span: rustc_span::Span| (get_pos(span.lo()), get_pos(span.hi()));
let get_line = |bytepos: rustc_span::BytePos| file.lookup_line(bytepos).unwrap();
let get_lines = |span: rustc_span::Span| (get_line(span.lo()), get_line(span.hi()));
if let Some(file_path) = file_path {
let abs_path = fs::canonicalize(file_path.clone()).unwrap();
let cx = &self.cx;
let enclosing_item_span =
self.tcx.hir().span_with_body(self.tcx.hir().get_parent_item(ex.hir_id));
assert!(enclosing_item_span.contains(span));
let location = CallLocation {
call_span: get_range(span),
enclosing_item_span: get_range(enclosing_item_span),
enclosing_item_lines: get_lines(enclosing_item_span),
};
let location = CallLocation::new(self.tcx, span, ex.hir_id, &file);
entries
.entry(abs_path)