Skip to content

Scrape code examples from examples/ directory for Rustdoc #9525

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 34 commits into from
Oct 28, 2021
Merged
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
8acf0e8
Initial support for scrape-examples
willcrichton May 11, 2021
3f9a4f2
Add repository URL
willcrichton May 11, 2021
a0122df
Share all examples across each unit in worksapce
willcrichton May 12, 2021
63c4346
Factor scraping and rendering into separate calls to rustdoc
willcrichton Jun 1, 2021
49a3a54
Add workspace root
willcrichton Jun 2, 2021
4ac6835
Fix for issue #9198
willcrichton Jun 3, 2021
b14a778
Change cargo strategy to go through rustdoc instead of doctest
willcrichton Jun 3, 2021
711539f
Add unstable notice for --scrape-examples
willcrichton Jun 3, 2021
6772991
Remove unnecessary collect
willcrichton Jun 3, 2021
5ed35a4
Fix test
willcrichton Jun 15, 2021
d19cfd2
Fix breakage
willcrichton Aug 25, 2021
48056e5
Allow scraping examples from library
willcrichton Aug 26, 2021
ff13eb5
Add comments
willcrichton Aug 27, 2021
0c8e1f8
Fix documentation
willcrichton Aug 27, 2021
0b2e293
Add tracking issue and unstable documentation
willcrichton Sep 14, 2021
b9b39a6
Pass metadata flag to rustdoc, ensure that Doc and Check units have s…
willcrichton Sep 21, 2021
dbcabc7
Change metadata strategy to extract hash from nested compilation
willcrichton Sep 21, 2021
0792cde
Revert change to lift doc units
willcrichton Sep 21, 2021
82d937e
Remove references to json
willcrichton Sep 21, 2021
70f3821
Change scraping strategy to embed in existing unit graph, add Compile…
willcrichton Oct 12, 2021
d29ac15
Remove unused code
willcrichton Oct 12, 2021
8331d7d
Remove more unused code
willcrichton Oct 12, 2021
223adac
Formatting
willcrichton Oct 12, 2021
19c8f05
Fix issues compiling build scripts
willcrichton Oct 13, 2021
4705566
Add documentation and a test
willcrichton Oct 13, 2021
17c6df7
Remove references to "lib" argument
willcrichton Oct 13, 2021
8b06a0f
Update rustdoc tests with -Cmetadata flag
willcrichton Oct 13, 2021
e52a9d9
Add comments and todos
willcrichton Oct 14, 2021
b948fc8
Add test / documentation for scrape-examples cycle-avoidance, add map…
willcrichton Oct 14, 2021
e4a65b9
Fix several bugs when checking wasmtime repo:
willcrichton Oct 27, 2021
0deeea8
Remove unnecessary clones, document out_dir
willcrichton Oct 27, 2021
1120957
Change scraping to apply to all workspace packages instead of just
willcrichton Oct 28, 2021
0a2382b
Formatting
willcrichton Oct 28, 2021
33718c7
Fix repeated warning with two calls to to_package_id_specs
willcrichton Oct 28, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 36 additions & 1 deletion src/bin/cargo/commands/doc.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use crate::command_prelude::*;

use cargo::ops::{self, DocOptions};
use anyhow::anyhow;
use cargo::ops::{self, CompileFilter, DocOptions, FilterRule, LibRule};

pub fn cli() -> App {
subcommand("doc")
@@ -19,6 +20,13 @@ pub fn cli() -> App {
)
.arg(opt("no-deps", "Don't build documentation for dependencies"))
.arg(opt("document-private-items", "Document private items"))
.arg(
opt(
"scrape-examples",
"Scrape examples to include as function documentation",
)
.value_name("FLAGS"),
)
.arg_jobs()
.arg_targets_lib_bin_example(
"Document only this package's library",
@@ -48,6 +56,33 @@ pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult {
args.compile_options(config, mode, Some(&ws), ProfileChecking::Custom)?;
compile_opts.rustdoc_document_private_items = args.is_present("document-private-items");

// TODO(wcrichto): move scrape example configuration into Cargo.toml before stabilization
// See: https://github.com/rust-lang/cargo/pull/9525#discussion_r728470927
compile_opts.rustdoc_scrape_examples = match args.value_of("scrape-examples") {
Some(s) => Some(match s {
"all" => CompileFilter::new_all_targets(),
"examples" => CompileFilter::new(
LibRule::False,
FilterRule::none(),
FilterRule::none(),
FilterRule::All,
FilterRule::none(),
),
_ => {
return Err(CliError::from(anyhow!(
r#"--scrape-examples must take "all" or "examples" as an argument"#
)));
}
}),
None => None,
};

if compile_opts.rustdoc_scrape_examples.is_some() {
config
.cli_unstable()
.fail_if_stable_opt("--scrape-examples", 9910)?;
}

let doc_opts = DocOptions {
open_result: args.is_present("open"),
compile_opts,
8 changes: 8 additions & 0 deletions src/cargo/core/compiler/build_config.rs
Original file line number Diff line number Diff line change
@@ -149,6 +149,8 @@ pub enum CompileMode {
Doc { deps: bool },
/// A target that will be tested with `rustdoc`.
Doctest,
/// An example or library that will be scraped for function calls by `rustdoc`.
Docscrape,
/// A marker for Units that represent the execution of a `build.rs` script.
RunCustomBuild,
}
@@ -166,6 +168,7 @@ impl ser::Serialize for CompileMode {
Bench => "bench".serialize(s),
Doc { .. } => "doc".serialize(s),
Doctest => "doctest".serialize(s),
Docscrape => "docscrape".serialize(s),
RunCustomBuild => "run-custom-build".serialize(s),
}
}
@@ -187,6 +190,11 @@ impl CompileMode {
self == CompileMode::Doctest
}

/// Returns `true` if this is scraping examples for documentation.
pub fn is_doc_scrape(self) -> bool {
self == CompileMode::Docscrape
}

/// Returns `true` if this is any type of test (test, benchmark, doc test, or
/// check test).
pub fn is_any_test(self) -> bool {
5 changes: 5 additions & 0 deletions src/cargo/core/compiler/build_context/mod.rs
Original file line number Diff line number Diff line change
@@ -47,6 +47,9 @@ pub struct BuildContext<'a, 'cfg> {
/// The dependency graph of units to compile.
pub unit_graph: UnitGraph,

/// Reverse-dependencies of documented units, used by the rustdoc --scrape-examples flag.
pub scrape_units: Vec<Unit>,

/// The list of all kinds that are involved in this build
pub all_kinds: HashSet<CompileKind>,
}
@@ -61,6 +64,7 @@ impl<'a, 'cfg> BuildContext<'a, 'cfg> {
target_data: RustcTargetData<'cfg>,
roots: Vec<Unit>,
unit_graph: UnitGraph,
scrape_units: Vec<Unit>,
) -> CargoResult<BuildContext<'a, 'cfg>> {
let all_kinds = unit_graph
.keys()
@@ -79,6 +83,7 @@ impl<'a, 'cfg> BuildContext<'a, 'cfg> {
target_data,
roots,
unit_graph,
scrape_units,
all_kinds,
})
}
5 changes: 4 additions & 1 deletion src/cargo/core/compiler/build_context/target_info.rs
Original file line number Diff line number Diff line change
@@ -461,7 +461,10 @@ impl TargetInfo {
}
}
CompileMode::Check { .. } => Ok((vec![FileType::new_rmeta()], Vec::new())),
CompileMode::Doc { .. } | CompileMode::Doctest | CompileMode::RunCustomBuild => {
CompileMode::Doc { .. }
| CompileMode::Doctest
| CompileMode::Docscrape
| CompileMode::RunCustomBuild => {
panic!("asked for rustc output for non-rustc mode")
}
}
15 changes: 14 additions & 1 deletion src/cargo/core/compiler/context/compilation_files.rs
Original file line number Diff line number Diff line change
@@ -191,7 +191,9 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
/// Returns the directory where the artifacts for the given unit are
/// initially created.
pub fn out_dir(&self, unit: &Unit) -> PathBuf {
if unit.mode.is_doc() {
// Docscrape units need to have doc/ set as the out_dir so sources for reverse-dependencies
// will be put into doc/ and not into deps/ where the *.examples files are stored.
if unit.mode.is_doc() || unit.mode.is_doc_scrape() {
self.layout(unit.kind).doc().to_path_buf()
} else if unit.mode.is_doc_test() {
panic!("doc tests do not have an out dir");
@@ -417,6 +419,17 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
// but Cargo does not know about that.
vec![]
}
CompileMode::Docscrape => {
let path = self
.deps_dir(unit)
.join(format!("{}.examples", unit.buildkey()));
vec![OutputFile {
path,
hardlink: None,
export_path: None,
flavor: FileFlavor::Normal,
}]
}
CompileMode::Test
| CompileMode::Build
| CompileMode::Bench
42 changes: 42 additions & 0 deletions src/cargo/core/compiler/context/mod.rs
Original file line number Diff line number Diff line change
@@ -81,6 +81,10 @@ pub struct Context<'a, 'cfg> {
/// compilation is happening (only object, only bitcode, both, etc), and is
/// precalculated early on.
pub lto: HashMap<Unit, Lto>,

/// Map of Doc/Docscrape units to metadata for their -Cmetadata flag.
/// See Context::find_metadata_units for more details.
pub metadata_for_doc_units: HashMap<Unit, Metadata>,
}

impl<'a, 'cfg> Context<'a, 'cfg> {
@@ -121,6 +125,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
rustc_clients: HashMap::new(),
pipelining,
lto: HashMap::new(),
metadata_for_doc_units: HashMap::new(),
})
}

@@ -135,6 +140,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
self.prepare()?;
custom_build::build_map(&mut self)?;
self.check_collisions()?;
self.compute_metadata_for_doc_units();

// We need to make sure that if there were any previous docs
// already compiled, they were compiled with the same Rustc version that we're currently
@@ -614,4 +620,40 @@ impl<'a, 'cfg> Context<'a, 'cfg> {

Ok(client)
}

/// Finds metadata for Doc/Docscrape units.
///
/// rustdoc needs a -Cmetadata flag in order to recognize StableCrateIds that refer to
/// items in the crate being documented. The -Cmetadata flag used by reverse-dependencies
/// will be the metadata of the Cargo unit that generated the current library's rmeta file,
/// which should be a Check unit.
///
/// If the current crate has reverse-dependencies, such a Check unit should exist, and so
/// we use that crate's metadata. If not, we use the crate's Doc unit so at least examples
/// scraped from the current crate can be used when documenting the current crate.
pub fn compute_metadata_for_doc_units(&mut self) {
for unit in self.bcx.unit_graph.keys() {
if !unit.mode.is_doc() && !unit.mode.is_doc_scrape() {
continue;
}

let matching_units = self
.bcx
.unit_graph
.keys()
.filter(|other| {
unit.pkg == other.pkg
&& unit.target == other.target
&& !other.mode.is_doc_scrape()
})
.collect::<Vec<_>>();
let metadata_unit = matching_units
.iter()
.find(|other| other.mode.is_check())
.or_else(|| matching_units.iter().find(|other| other.mode.is_doc()))
.unwrap_or(&unit);
self.metadata_for_doc_units
.insert(unit.clone(), self.files().metadata(metadata_unit));
}
}
}
38 changes: 37 additions & 1 deletion src/cargo/core/compiler/mod.rs
Original file line number Diff line number Diff line change
@@ -165,7 +165,7 @@ fn compile<'cfg>(
let force = exec.force_rebuild(unit) || force_rebuild;
let mut job = fingerprint::prepare_target(cx, unit, force)?;
job.before(if job.freshness() == Freshness::Dirty {
let work = if unit.mode.is_doc() {
let work = if unit.mode.is_doc() || unit.mode.is_doc_scrape() {
rustdoc(cx, unit)?
} else {
rustc(cx, unit, exec)?
@@ -647,6 +647,42 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
rustdoc.args(args);
}

let metadata = cx.metadata_for_doc_units[&unit];
rustdoc.arg("-C").arg(format!("metadata={}", metadata));

let scrape_output_path = |unit: &Unit| -> CargoResult<PathBuf> {
let output_dir = cx.files().deps_dir(unit);
Ok(output_dir.join(format!("{}.examples", unit.buildkey())))
};

if unit.mode.is_doc_scrape() {
debug_assert!(cx.bcx.scrape_units.contains(unit));

rustdoc.arg("-Zunstable-options");

rustdoc
.arg("--scrape-examples-output-path")
.arg(scrape_output_path(unit)?);

// Only scrape example for items from crates in the workspace, to reduce generated file size
for pkg in cx.bcx.ws.members() {
rustdoc
.arg("--scrape-examples-target-crate")
.arg(pkg.name());
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How does this interact if you do something like cargo doc -p foo -p bar where the "root set" is larger than one? Presumably the documenation for foo wouldn't want to get --scrape-examples-target-crate bar, right?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the other way around. If you have a file examples/ex.rs like

fn main() {
  foo::f();
  bar::f();
  baz::f();
  Vec::new();
}

Then the --scrape-examples-target-crate indicates which calls to save as examples. So if you do -p foo -p bar then only foo::f() and bar::f() will be saved, and not baz::f() or Vec::new().

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if foo and bar are unrelated crates though? In that what if they're both leaves of the dependency tree (or distinct roots depending on your tree perspective)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure what your concern is. This flag is being passed to reverse-dependencies being scraped, not to packages being documented. So this isn't saying "documentation for crate foo should include examples for crate bar". It's saying "examples for foo and bar should be scraped from example, and those examples will be available while documenting foo and bar."

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm still slightly concerned about this but I don't know enough about these new rustdoc flags to really bottom our my concern so "if it works it works"

} else if cx.bcx.scrape_units.len() > 0 && cx.bcx.ws.is_member(&unit.pkg) {
// We only pass scraped examples to packages in the workspace
// since examples are only coming from reverse-dependencies of workspace packages

rustdoc.arg("-Zunstable-options");

for scrape_unit in &cx.bcx.scrape_units {
rustdoc
.arg("--with-examples")
.arg(scrape_output_path(scrape_unit)?);
}
}

build_deps_args(&mut rustdoc, cx, unit)?;
rustdoc::add_root_urls(cx, unit, &mut rustdoc)?;

1 change: 1 addition & 0 deletions src/cargo/core/compiler/timings.rs
Original file line number Diff line number Diff line change
@@ -176,6 +176,7 @@ impl<'cfg> Timings<'cfg> {
CompileMode::Bench => target.push_str(" (bench)"),
CompileMode::Doc { .. } => target.push_str(" (doc)"),
CompileMode::Doctest => target.push_str(" (doc test)"),
CompileMode::Docscrape => target.push_str(" (doc scrape)"),
CompileMode::RunCustomBuild => target.push_str(" (run)"),
}
let unit_time = UnitTime {
33 changes: 32 additions & 1 deletion src/cargo/core/compiler/unit_dependencies.rs
Original file line number Diff line number Diff line change
@@ -47,6 +47,7 @@ struct State<'a, 'cfg> {
target_data: &'a RustcTargetData<'cfg>,
profiles: &'a Profiles,
interner: &'a UnitInterner,
scrape_units: &'a [Unit],

/// A set of edges in `unit_dependencies` where (a, b) means that the
/// dependency from a to b was added purely because it was a dev-dependency.
@@ -61,6 +62,7 @@ pub fn build_unit_dependencies<'a, 'cfg>(
features: &'a ResolvedFeatures,
std_resolve: Option<&'a (Resolve, ResolvedFeatures)>,
roots: &[Unit],
scrape_units: &[Unit],
std_roots: &HashMap<CompileKind, Vec<Unit>>,
global_mode: CompileMode,
target_data: &'a RustcTargetData<'cfg>,
@@ -91,6 +93,7 @@ pub fn build_unit_dependencies<'a, 'cfg>(
target_data,
profiles,
interner,
scrape_units,
dev_dependency_edges: HashSet::new(),
};

@@ -253,6 +256,7 @@ fn compute_deps(
if !dep.is_transitive()
&& !unit.target.is_test()
&& !unit.target.is_example()
&& !unit.mode.is_doc_scrape()
&& !unit.mode.is_any_test()
{
return false;
@@ -467,6 +471,25 @@ fn compute_deps_doc(
if unit.target.is_bin() || unit.target.is_example() {
ret.extend(maybe_lib(unit, state, unit_for)?);
}

// Add all units being scraped for examples as a dependency of Doc units.
if state.ws.is_member(&unit.pkg) {
for scrape_unit in state.scrape_units.iter() {
// This needs to match the FeaturesFor used in cargo_compile::generate_targets.
let unit_for = UnitFor::new_host(scrape_unit.target.proc_macro());
deps_of(scrape_unit, state, unit_for)?;
ret.push(new_unit_dep(
state,
scrape_unit,
&scrape_unit.pkg,
&scrape_unit.target,
unit_for,
scrape_unit.kind,
scrape_unit.mode,
)?);
}
}

Ok(ret)
}

@@ -558,7 +581,7 @@ fn dep_build_script(
/// Choose the correct mode for dependencies.
fn check_or_build_mode(mode: CompileMode, target: &Target) -> CompileMode {
match mode {
CompileMode::Check { .. } | CompileMode::Doc { .. } => {
CompileMode::Check { .. } | CompileMode::Doc { .. } | CompileMode::Docscrape => {
if target.for_host() {
// Plugin and proc macro targets should be compiled like
// normal.
@@ -695,6 +718,14 @@ fn connect_run_custom_build_deps(state: &mut State<'_, '_>) {
&& other.unit.target.is_linkable()
&& other.unit.pkg.manifest().links().is_some()
})
// Avoid cycles when using the doc --scrape-examples feature:
// Say a workspace has crates A and B where A has a build-dependency on B.
// The Doc units for A and B will have a dependency on the Docscrape for both A and B.
// So this would add a dependency from B-build to A-build, causing a cycle:
// B (build) -> A (build) -> B(build)
// See the test scrape_examples_avoid_build_script_cycle for a concrete example.
// To avoid this cycle, we filter out the B -> A (docscrape) dependency.
.filter(|(_parent, other)| !other.unit.mode.is_doc_scrape())
// Skip dependencies induced via dev-dependencies since
// connections between `links` and build scripts only happens
// via normal dependencies. Otherwise since dev-dependencies can
4 changes: 3 additions & 1 deletion src/cargo/core/profiles.rs
Original file line number Diff line number Diff line change
@@ -323,7 +323,9 @@ impl Profiles {
(InternedString::new("dev"), None)
}
}
CompileMode::Doc { .. } => (InternedString::new("doc"), None),
CompileMode::Doc { .. } | CompileMode::Docscrape => {
(InternedString::new("doc"), None)
}
}
} else {
(self.requested_profile, None)
102 changes: 81 additions & 21 deletions src/cargo/ops/cargo_compile.rs
Original file line number Diff line number Diff line change
@@ -76,6 +76,9 @@ pub struct CompileOptions {
/// Whether the `--document-private-items` flags was specified and should
/// be forwarded to `rustdoc`.
pub rustdoc_document_private_items: bool,
/// Whether the `--scrape-examples` flag was specified and build flags for
/// examples should be forwarded to `rustdoc`.
pub rustdoc_scrape_examples: Option<CompileFilter>,
/// Whether the build process should check the minimum Rust version
/// defined in the cargo metadata for a crate.
pub honor_rust_version: bool,
@@ -94,12 +97,13 @@ impl<'a> CompileOptions {
target_rustc_args: None,
local_rustdoc_args: None,
rustdoc_document_private_items: false,
rustdoc_scrape_examples: None,
honor_rust_version: true,
})
}
}

#[derive(Clone, PartialEq, Eq, Debug)]
#[derive(PartialEq, Eq, Debug)]
pub enum Packages {
Default,
All,
@@ -334,6 +338,7 @@ pub fn create_bcx<'a, 'cfg>(
ref target_rustc_args,
ref local_rustdoc_args,
rustdoc_document_private_items,
ref rustdoc_scrape_examples,
honor_rust_version,
} = *options;
let config = ws.config();
@@ -351,7 +356,7 @@ pub fn create_bcx<'a, 'cfg>(
)?;
}
}
CompileMode::Doc { .. } | CompileMode::Doctest => {
CompileMode::Doc { .. } | CompileMode::Doctest | CompileMode::Docscrape => {
if std::env::var("RUSTDOC_FLAGS").is_ok() {
config.shell().warn(
"Cargo does not read `RUSTDOC_FLAGS` environment variable. Did you mean `RUSTDOCFLAGS`?"
@@ -363,8 +368,16 @@ pub fn create_bcx<'a, 'cfg>(

let target_data = RustcTargetData::new(ws, &build_config.requested_kinds)?;

let specs = spec.to_package_id_specs(ws)?;
let has_dev_units = if filter.need_dev_deps(build_config.mode) {
let all_packages = &Packages::All;
let need_reverse_dependencies = rustdoc_scrape_examples.is_some();
let full_specs = if need_reverse_dependencies {
all_packages
} else {
spec
};

let resolve_specs = full_specs.to_package_id_specs(ws)?;
let has_dev_units = if filter.need_dev_deps(build_config.mode) || need_reverse_dependencies {
HasDevUnits::Yes
} else {
HasDevUnits::No
@@ -374,7 +387,7 @@ pub fn create_bcx<'a, 'cfg>(
&target_data,
&build_config.requested_kinds,
cli_features,
&specs,
&resolve_specs,
has_dev_units,
crate::core::resolver::features::ForceAllTargets::No,
)?;
@@ -408,6 +421,11 @@ pub fn create_bcx<'a, 'cfg>(
// Find the packages in the resolver that the user wants to build (those
// passed in with `-p` or the defaults from the workspace), and convert
// Vec<PackageIdSpec> to a Vec<PackageId>.
let specs = if need_reverse_dependencies {
spec.to_package_id_specs(ws)?
} else {
resolve_specs.clone()
};
let to_build_ids = resolve.specs_to_ids(&specs)?;
// Now get the `Package` for each `PackageId`. This may trigger a download
// if the user specified `-p` for a dependency that is not downloaded.
@@ -487,6 +505,30 @@ pub fn create_bcx<'a, 'cfg>(
interner,
)?;

let mut scrape_units = match rustdoc_scrape_examples {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So this is one place where the final implementation might diverge significantly in a way that's better to implement nowadays. Whether or not this is necessary to do at this time I think should probably get some input from other folks.

Right now with the CLI argument I think this is the proper place to do this (albeit wonky to call generate_targets twice). Alternatively what I view is probably the more long-term way we'll implement this is with Cargo.toml-based configuration of the package. I'm imagining something like:

[lib]
doc-scrape-examples = true # the default is true, so this isn't necessary

[[example]]
name = "scratch"
doc-scrape-examples = false # in case you think this is sloppy code not to be referenced

In a world like that with manifest-based configuration I don't think that this is the appropriate location to manage this. With manifest-based configuration I think you'd process and make new units as part of unit_dependencies.rs. Basically when calculating the dependencies of a doc unit we'd explicitly include the doc-scrape units, which would then recursively include all their own dependencies as well.

The end result I think roughly boils down to the same thing in that you can configure what's scraped and additionally you the representation in Cargo is "it's all in the unit graph". Not having an explicit list of scrape_units though would make fn rustdoc in the compile section a bit different though. This I think is a good thing because we try to avoid "package global state" in the compile graph since it doesn't really map well to commands which want to work over the whole workspace or things like that.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh and in terms of a "how do we roll this out" story I would imagine that you would have to specify cargo-features = ["doc-scrape-examples"] to actually specify doc-scrape-examples in a manifest, and we would default it to false initially. Once we stabilize this we would then roll out the default of false to stable Rust, and then when we're satisfied it's had testing and such we'd roll out an update to the default being true instead of false

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok this sounds like a good plan. I'll add a link to this comment in the source.

Some(scrape_filter) => {
let to_build_ids = resolve.specs_to_ids(&resolve_specs)?;
let to_builds = pkg_set.get_many(to_build_ids)?;
let mode = CompileMode::Docscrape;

generate_targets(
ws,
&to_builds,
scrape_filter,
&build_config.requested_kinds,
explicit_host_kind,
mode,
&resolve,
&workspace_resolve,
&resolved_features,
&pkg_set,
&profiles,
interner,
)?
}
None => Vec::new(),
};

let std_roots = if let Some(crates) = &config.cli_unstable().build_std {
// Only build libtest if it looks like it is needed.
let mut crates = crates.clone();
@@ -521,6 +563,7 @@ pub fn create_bcx<'a, 'cfg>(
&resolved_features,
std_resolve_features.as_ref(),
&units,
&scrape_units,
&std_roots,
build_config.mode,
&target_data,
@@ -542,10 +585,17 @@ pub fn create_bcx<'a, 'cfg>(
// Rebuild the unit graph, replacing the explicit host targets with
// CompileKind::Host, merging any dependencies shared with build
// dependencies.
let new_graph = rebuild_unit_graph_shared(interner, unit_graph, &units, explicit_host_kind);
let new_graph = rebuild_unit_graph_shared(
interner,
unit_graph,
&units,
&scrape_units,
explicit_host_kind,
);
// This would be nicer with destructuring assignment.
units = new_graph.0;
unit_graph = new_graph.1;
scrape_units = new_graph.1;
unit_graph = new_graph.2;
}

let mut extra_compiler_args = HashMap::new();
@@ -560,6 +610,7 @@ pub fn create_bcx<'a, 'cfg>(
}
extra_compiler_args.insert(units[0].clone(), args);
}

for unit in &units {
if unit.mode.is_doc() || unit.mode.is_doc_test() {
let mut extra_args = local_rustdoc_args.clone();
@@ -621,6 +672,7 @@ pub fn create_bcx<'a, 'cfg>(
target_data,
units,
unit_graph,
scrape_units,
)?;

Ok(bcx)
@@ -742,17 +794,18 @@ impl CompileFilter {
match mode {
CompileMode::Test | CompileMode::Doctest | CompileMode::Bench => true,
CompileMode::Check { test: true } => true,
CompileMode::Build | CompileMode::Doc { .. } | CompileMode::Check { test: false } => {
match *self {
CompileFilter::Default { .. } => false,
CompileFilter::Only {
ref examples,
ref tests,
ref benches,
..
} => examples.is_specific() || tests.is_specific() || benches.is_specific(),
}
}
CompileMode::Build
| CompileMode::Doc { .. }
| CompileMode::Docscrape
| CompileMode::Check { test: false } => match *self {
CompileFilter::Default { .. } => false,
CompileFilter::Only {
ref examples,
ref tests,
ref benches,
..
} => examples.is_specific() || tests.is_specific() || benches.is_specific(),
},
CompileMode::RunCustomBuild => panic!("Invalid mode"),
}
}
@@ -1342,7 +1395,9 @@ fn filter_default_targets(targets: &[Target], mode: CompileMode) -> Vec<&Target>
})
.collect()
}
CompileMode::Doctest | CompileMode::RunCustomBuild => panic!("Invalid mode {:?}", mode),
CompileMode::Doctest | CompileMode::Docscrape | CompileMode::RunCustomBuild => {
panic!("Invalid mode {:?}", mode)
}
}
}

@@ -1454,8 +1509,9 @@ fn rebuild_unit_graph_shared(
interner: &UnitInterner,
unit_graph: UnitGraph,
roots: &[Unit],
scrape_units: &[Unit],
to_host: CompileKind,
) -> (Vec<Unit>, UnitGraph) {
) -> (Vec<Unit>, Vec<Unit>, UnitGraph) {
let mut result = UnitGraph::new();
// Map of the old unit to the new unit, used to avoid recursing into units
// that have already been computed to improve performance.
@@ -1466,7 +1522,11 @@ fn rebuild_unit_graph_shared(
traverse_and_share(interner, &mut memo, &mut result, &unit_graph, root, to_host)
})
.collect();
(new_roots, result)
let new_scrape_units = scrape_units
.iter()
.map(|unit| memo.get(unit).unwrap().clone())
.collect();
(new_roots, new_scrape_units, result)
}

/// Recursive function for rebuilding the graph.
1 change: 1 addition & 0 deletions src/cargo/ops/cargo_package.rs
Original file line number Diff line number Diff line change
@@ -765,6 +765,7 @@ fn run_verify(
target_rustc_args: rustc_args,
local_rustdoc_args: None,
rustdoc_document_private_items: false,
rustdoc_scrape_examples: None,
honor_rust_version: true,
},
&exec,
1 change: 1 addition & 0 deletions src/cargo/util/command_prelude.rs
Original file line number Diff line number Diff line change
@@ -544,6 +544,7 @@ pub trait ArgMatchesExt {
target_rustc_args: None,
local_rustdoc_args: None,
rustdoc_document_private_items: false,
rustdoc_scrape_examples: None,
honor_rust_version: !self._is_present("ignore-rust-version"),
};

18 changes: 17 additions & 1 deletion src/doc/src/reference/unstable.md
Original file line number Diff line number Diff line change
@@ -1207,7 +1207,7 @@ for the appropriate target and influenced by any other RUSTFLAGS.
* Tracking Issue: [#9778](https://github.com/rust-lang/cargo/issues/9778)
* PR: [#9627](https://github.com/rust-lang/cargo/pull/9627)

The `different-binary-name` feature allows setting the filename of the binary without having to obey the
The `different-binary-name` feature allows setting the filename of the binary without having to obey the
restrictions placed on crate names. For example, the crate name must use only `alphanumeric` characters
or `-` or `_`, and cannot be empty.

@@ -1378,7 +1378,23 @@ The 2021 edition has been stabilized in the 1.56 release.
See the [`edition` field](manifest.md#the-edition-field) for more information on setting the edition.
See [`cargo fix --edition`](../commands/cargo-fix.md) and [The Edition Guide](../../edition-guide/index.html) for more information on migrating existing projects.


### Custom named profiles

Custom named profiles have been stabilized in the 1.57 release. See the
[profiles chapter](profiles.md#custom-profiles) for more information.


### scrape-examples

* RFC: [#3123](https://github.com/rust-lang/rfcs/pull/3123)
* Tracking Issue: [#9910](https://github.com/rust-lang/cargo/issues/9910)

The `--scrape-examples` argument to the `doc` command tells Rustdoc to search
crates in the current workspace for calls to functions. Those call-sites are then
included as documentation. The flag can take an argument of `all` or `examples`
which configures which crate in the workspace to analyze for examples. For instance:

```
cargo doc -Z unstable-options --scrape-examples examples
```
149 changes: 149 additions & 0 deletions tests/testsuite/doc.rs
Original file line number Diff line number Diff line change
@@ -2148,3 +2148,152 @@ fn doc_fingerprint_unusual_behavior() {
assert!(build_doc.join("somefile").exists());
assert!(real_doc.join("somefile").exists());
}

#[cargo_test]
fn scrape_examples_basic() {
if !is_nightly() {
// --scrape-examples is unstable
return;
}

let p = project()
.file(
"Cargo.toml",
r#"
[package]
name = "foo"
version = "0.0.1"
authors = []
"#,
)
.file("examples/ex.rs", "fn main() { foo::foo(); }")
.file("src/lib.rs", "pub fn foo() {}\npub fn bar() { foo(); }")
.build();

p.cargo("doc -Zunstable-options --scrape-examples all")
.masquerade_as_nightly_cargo()
.with_stderr(
"\
[..] foo v0.0.1 ([CWD])
[..] foo v0.0.1 ([CWD])
[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]
",
)
.run();

let doc_html = p.read_file("target/doc/foo/fn.foo.html");
assert!(doc_html.contains("Examples found in repository"));
assert!(doc_html.contains("More examples"));

// Ensure that the reverse-dependency has its sources generated
assert!(p.build_dir().join("doc/src/ex/ex.rs.html").exists());
}

#[cargo_test]
fn scrape_examples_avoid_build_script_cycle() {
if !is_nightly() {
// --scrape-examples is unstable
return;
}

let p = project()
// package with build dependency
.file(
"Cargo.toml",
r#"
[package]
name = "foo"
version = "0.0.1"
authors = []
links = "foo"
[workspace]
members = ["bar"]
[build-dependencies]
bar = {path = "bar"}
"#,
)
.file("src/lib.rs", "")
.file("build.rs", "fn main(){}")
// dependency
.file(
"bar/Cargo.toml",
r#"
[package]
name = "bar"
version = "0.0.1"
authors = []
links = "bar"
"#,
)
.file("bar/src/lib.rs", "")
.file("bar/build.rs", "fn main(){}")
.build();

p.cargo("doc --all -Zunstable-options --scrape-examples all")
.masquerade_as_nightly_cargo()
.run();
}

#[cargo_test]
fn scrape_examples_complex_reverse_dependencies() {
if !is_nightly() {
// --scrape-examples is unstable
return;
}

let p = project()
.file(
"Cargo.toml",
r#"
[package]
name = "foo"
version = "0.0.1"
authors = []
[dev-dependencies]
a = {path = "a", features = ["feature"]}
b = {path = "b"}
[workspace]
members = ["b"]
"#,
)
.file("src/lib.rs", "")
.file("examples/ex.rs", "fn main() { a::f(); }")
.file(
"a/Cargo.toml",
r#"
[package]
name = "a"
version = "0.0.1"
authors = []
[lib]
proc-macro = true
[dependencies]
b = {path = "../b"}
[features]
feature = []
"#,
)
.file("a/src/lib.rs", "#[cfg(feature)] pub fn f();")
.file(
"b/Cargo.toml",
r#"
[package]
name = "b"
version = "0.0.1"
authors = []
"#,
)
.file("b/src/lib.rs", "")
.build();

p.cargo("doc -Zunstable-options --scrape-examples all")
.masquerade_as_nightly_cargo()
.run();
}
4 changes: 4 additions & 0 deletions tests/testsuite/rustdoc.rs
Original file line number Diff line number Diff line change
@@ -32,6 +32,7 @@ fn rustdoc_args() {
-o [CWD]/target/doc \
[..] \
--cfg=foo \
-C metadata=[..] \
-L dependency=[CWD]/target/debug/deps [..]`
[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]
",
@@ -83,6 +84,7 @@ fn rustdoc_foo_with_bar_dependency() {
-o [CWD]/target/doc \
[..] \
--cfg=foo \
-C metadata=[..] \
-L dependency=[CWD]/target/debug/deps \
--extern [..]`
[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]
@@ -122,6 +124,7 @@ fn rustdoc_only_bar_dependency() {
-o [CWD]/target/doc \
[..] \
--cfg=foo \
-C metadata=[..] \
-L dependency=[CWD]/target/debug/deps [..]`
[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]
",
@@ -144,6 +147,7 @@ fn rustdoc_same_name_documents_lib() {
-o [CWD]/target/doc \
[..] \
--cfg=foo \
-C metadata=[..] \
-L dependency=[CWD]/target/debug/deps [..]`
[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]
",