Skip to content

Rewrite SyntaxEnv #11228

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 3, 2014
+129 −247
Merged

Rewrite SyntaxEnv #11228

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
266 changes: 88 additions & 178 deletions src/libsyntax/ext/base.rs
View file
Original file line number Diff line number Diff line change
@@ -156,57 +156,48 @@ pub enum SyntaxExtension {
// The SyntaxEnv is the environment that's threaded through the expansion
// of macros. It contains bindings for macros, and also a special binding
// for " block" (not a legal identifier) that maps to a BlockInfo
pub type SyntaxEnv = @mut MapChain<Name, Transformer>;

// Transformer : the codomain of SyntaxEnvs

pub enum Transformer {
// this identifier maps to a syntax extension or macro
SE(SyntaxExtension),
// blockinfo : this is ... well, it's simpler than threading
// another whole data stack-structured data structure through
// expansion. Basically, there's an invariant that every
// map must contain a binding for " block".
BlockInfo(BlockInfo)
}
pub type SyntaxEnv = MapChain<Name, SyntaxExtension>;

pub struct BlockInfo {
// should macros escape from this scope?
macros_escape : bool,
// what are the pending renames?
pending_renames : @mut RenameList
pending_renames : RenameList
}

impl BlockInfo {
pub fn new() -> BlockInfo {
BlockInfo {
macros_escape: false,
pending_renames: ~[]
}
}
}

// a list of ident->name renamings
type RenameList = ~[(ast::Ident,Name)];
pub type RenameList = ~[(ast::Ident,Name)];

// The base map of methods for expanding syntax extension
// AST nodes into full ASTs
pub fn syntax_expander_table() -> SyntaxEnv {
// utility function to simplify creating NormalTT syntax extensions
fn builtin_normal_tt_no_ctxt(f: SyntaxExpanderTTFunNoCtxt)
-> @Transformer {
@SE(NormalTT(@SyntaxExpanderTT{
-> SyntaxExtension {
NormalTT(@SyntaxExpanderTT{
expander: SyntaxExpanderTTExpanderWithoutContext(f),
span: None,
} as @SyntaxExpanderTTTrait,
None))
None)
}

let mut syntax_expanders = HashMap::new();
// NB identifier starts with space, and can't conflict with legal idents
syntax_expanders.insert(intern(&" block"),
@BlockInfo(BlockInfo{
macros_escape : false,
pending_renames : @mut ~[]
}));
let mut syntax_expanders = MapChain::new();
syntax_expanders.insert(intern(&"macro_rules"),
@SE(IdentTT(@SyntaxExpanderTTItem {
IdentTT(@SyntaxExpanderTTItem {
expander: SyntaxExpanderTTItemExpanderWithContext(
ext::tt::macro_rules::add_new_extension),
span: None,
} as @SyntaxExpanderTTItemTrait,
None)));
None));
syntax_expanders.insert(intern(&"fmt"),
builtin_normal_tt_no_ctxt(
ext::fmt::expand_syntax_ext));
@@ -232,8 +223,7 @@ pub fn syntax_expander_table() -> SyntaxEnv {
builtin_normal_tt_no_ctxt(
ext::log_syntax::expand_syntax_ext));
syntax_expanders.insert(intern(&"deriving"),
@SE(ItemDecorator(
ext::deriving::expand_meta_deriving)));
ItemDecorator(ext::deriving::expand_meta_deriving));

// Quasi-quoting expanders
syntax_expanders.insert(intern(&"quote_tokens"),
@@ -288,7 +278,7 @@ pub fn syntax_expander_table() -> SyntaxEnv {
syntax_expanders.insert(intern(&"trace_macros"),
builtin_normal_tt_no_ctxt(
ext::trace_macros::expand_trace_macros));
MapChain::new(~syntax_expanders)
syntax_expanders
}

// One of these is made during expansion and incrementally updated as we go;
@@ -299,11 +289,6 @@ pub struct ExtCtxt {
cfg: ast::CrateConfig,
backtrace: Option<@ExpnInfo>,

// These two @mut's should really not be here,
// but the self types for CtxtRepr are all wrong
// and there are bugs in the code for object
// types that make this hard to get right at the
// moment. - nmatsakis
mod_path: ~[ast::Ident],
trace_mac: bool
}
@@ -325,7 +310,7 @@ impl ExtCtxt {
match e.node {
ast::ExprMac(..) => {
let mut expander = expand::MacroExpander {
extsbox: @mut syntax_expander_table(),
extsbox: syntax_expander_table(),
cx: self,
};
e = expand::expand_expr(e, &mut expander);
@@ -460,11 +445,7 @@ pub fn get_exprs_from_tts(cx: &ExtCtxt,
// we want to implement the notion of a transformation
// environment.

// This environment maps Names to Transformers.
// Initially, this includes macro definitions and
// block directives.


// This environment maps Names to SyntaxExtensions.

// Actually, the following implementation is parameterized
// by both key and value types.
@@ -479,169 +460,98 @@ pub fn get_exprs_from_tts(cx: &ExtCtxt,
// able to refer to a macro that was added to an enclosing
// scope lexically later than the deeper scope.

// Note on choice of representation: I've been pushed to
// use a top-level managed pointer by some difficulties
// with pushing and popping functionally, and the ownership
// issues. As a result, the values returned by the table
// also need to be managed; the &'a ... type that Maps
// return won't work for things that need to get outside
// of that managed pointer. The easiest way to do this
// is just to insist that the values in the tables are
// managed to begin with.

// a transformer env is either a base map or a map on top
// of another chain.
pub enum MapChain<K,V> {
BaseMapChain(~HashMap<K,@V>),
ConsMapChain(~HashMap<K,@V>,@mut MapChain<K,V>)
// Only generic to make it easy to test
struct MapChainFrame<K, V> {
info: BlockInfo,
map: HashMap<K, V>,
}

// Only generic to make it easy to test
pub struct MapChain<K, V> {
priv chain: ~[MapChainFrame<K, V>],
}

// get the map from an env frame
impl <K: Eq + Hash + IterBytes + 'static, V: 'static> MapChain<K,V>{
// Constructor. I don't think we need a zero-arg one.
pub fn new(init: ~HashMap<K,@V>) -> @mut MapChain<K,V> {
@mut BaseMapChain(init)
impl<K: Hash+Eq, V> MapChain<K, V> {
pub fn new() -> MapChain<K, V> {
let mut map = MapChain { chain: ~[] };
map.push_frame();
map
}

// add a new frame to the environment (functionally)
pub fn push_frame (@mut self) -> @mut MapChain<K,V> {
@mut ConsMapChain(~HashMap::new() ,self)
pub fn push_frame(&mut self) {
self.chain.push(MapChainFrame {
info: BlockInfo::new(),
map: HashMap::new(),
});
}

// no need for pop, it'll just be functional.

// utility fn...

// ugh: can't get this to compile with mut because of the
// lack of flow sensitivity.
pub fn get_map<'a>(&'a self) -> &'a HashMap<K,@V> {
match *self {
BaseMapChain (~ref map) => map,
ConsMapChain (~ref map,_) => map
}
pub fn pop_frame(&mut self) {
assert!(self.chain.len() > 1, "too many pops on MapChain!");
self.chain.pop();
}

// traits just don't work anywhere...?
//impl Map<Name,SyntaxExtension> for MapChain {

pub fn contains_key (&self, key: &K) -> bool {
match *self {
BaseMapChain (ref map) => map.contains_key(key),
ConsMapChain (ref map,ref rest) =>
(map.contains_key(key)
|| rest.contains_key(key))
fn find_escape_frame<'a>(&'a mut self) -> &'a mut MapChainFrame<K, V> {
for (i, frame) in self.chain.mut_iter().enumerate().invert() {
if !frame.info.macros_escape || i == 0 {
return frame
}
}
}
// should each_key and each_value operate on shadowed
// names? I think not.
// delaying implementing this....
pub fn each_key (&self, _f: |&K| -> bool) {
fail!("unimplemented 2013-02-15T10:01");
}

pub fn each_value (&self, _f: |&V| -> bool) {
fail!("unimplemented 2013-02-15T10:02");
unreachable!()
}

// Returns a copy of the value that the name maps to.
// Goes down the chain 'til it finds one (or bottom out).
pub fn find (&self, key: &K) -> Option<@V> {
match self.get_map().find (key) {
Some(ref v) => Some(**v),
None => match *self {
BaseMapChain (_) => None,
ConsMapChain (_,ref rest) => rest.find(key)
pub fn find<'a>(&'a self, k: &K) -> Option<&'a V> {
for frame in self.chain.iter().invert() {
match frame.map.find(k) {
Some(v) => return Some(v),
None => {}
}
}
None
}

pub fn find_in_topmost_frame(&self, key: &K) -> Option<@V> {
let map = match *self {
BaseMapChain(ref map) => map,
ConsMapChain(ref map,_) => map
};
// strip one layer of indirection off the pointer.
map.find(key).map(|r| {*r})
}

// insert the binding into the top-level map
pub fn insert (&mut self, key: K, ext: @V) -> bool {
// can't abstract over get_map because of flow sensitivity...
match *self {
BaseMapChain (~ref mut map) => map.insert(key, ext),
ConsMapChain (~ref mut map,_) => map.insert(key,ext)
}
pub fn insert(&mut self, k: K, v: V) {
self.find_escape_frame().map.insert(k, v);
}
// insert the binding into the topmost frame for which the binding
// associated with 'n' exists and satisfies pred
// ... there are definitely some opportunities for abstraction
// here that I'm ignoring. (e.g., manufacturing a predicate on
// the maps in the chain, and using an abstract "find".
pub fn insert_into_frame(&mut self,
key: K,
ext: @V,
n: K,
pred: |&@V| -> bool) {
match *self {
BaseMapChain (~ref mut map) => {
if satisfies_pred(map,&n,pred) {
map.insert(key,ext);
} else {
fail!("expected map chain containing satisfying frame")
}
},
ConsMapChain (~ref mut map, rest) => {
if satisfies_pred(map,&n,|v|pred(v)) {
map.insert(key,ext);
} else {
rest.insert_into_frame(key,ext,n,pred)
}
}
}
}
}

// returns true if the binding for 'n' satisfies 'pred' in 'map'
fn satisfies_pred<K:Eq + Hash + IterBytes,
V>(
map: &mut HashMap<K,V>,
n: &K,
pred: |&V| -> bool)
-> bool {
match map.find(n) {
Some(ref v) => (pred(*v)),
None => false
pub fn info<'a>(&'a mut self) -> &'a mut BlockInfo {
&mut self.chain[self.chain.len()-1].info
}
}

#[cfg(test)]
mod test {
use super::MapChain;
use std::hashmap::HashMap;

#[test]
fn testenv() {
let mut a = HashMap::new();
a.insert (@"abc",@15);
let m = MapChain::new(~a);
m.insert (@"def",@16);
assert_eq!(m.find(&@"abc"),Some(@15));
assert_eq!(m.find(&@"def"),Some(@16));
assert_eq!(*(m.find(&@"abc").unwrap()),15);
assert_eq!(*(m.find(&@"def").unwrap()),16);
let n = m.push_frame();
// old bindings are still present:
assert_eq!(*(n.find(&@"abc").unwrap()),15);
assert_eq!(*(n.find(&@"def").unwrap()),16);
n.insert (@"def",@17);
// n shows the new binding
assert_eq!(*(n.find(&@"abc").unwrap()),15);
assert_eq!(*(n.find(&@"def").unwrap()),17);
// ... but m still has the old ones
assert_eq!(m.find(&@"abc"),Some(@15));
assert_eq!(m.find(&@"def"),Some(@16));
assert_eq!(*(m.find(&@"abc").unwrap()),15);
assert_eq!(*(m.find(&@"def").unwrap()),16);
let mut m = MapChain::new();
let (a,b,c,d) = ("a", "b", "c", "d");
m.insert(1, a);
assert_eq!(Some(&a), m.find(&1));

m.push_frame();
m.info().macros_escape = true;
m.insert(2, b);
assert_eq!(Some(&a), m.find(&1));
assert_eq!(Some(&b), m.find(&2));
m.pop_frame();

assert_eq!(Some(&a), m.find(&1));
assert_eq!(Some(&b), m.find(&2));

m.push_frame();
m.push_frame();
m.info().macros_escape = true;
m.insert(3, c);
assert_eq!(Some(&c), m.find(&3));
m.pop_frame();
assert_eq!(Some(&c), m.find(&3));
m.pop_frame();
assert_eq!(None, m.find(&3));

m.push_frame();
m.insert(4, d);
m.pop_frame();
assert_eq!(None, m.find(&4));
}
}
110 changes: 41 additions & 69 deletions src/libsyntax/ext/expand.rs
View file
Original file line number Diff line number Diff line change
@@ -53,13 +53,13 @@ pub fn expand_expr(e: @ast::Expr, fld: &mut MacroExpander) -> @ast::Expr {
let extname = &pth.segments[0].identifier;
let extnamestr = ident_to_str(extname);
// leaving explicit deref here to highlight unbox op:
match (*fld.extsbox).find(&extname.name) {
match fld.extsbox.find(&extname.name) {
None => {
fld.cx.span_fatal(
pth.span,
format!("macro undefined: '{}'", extnamestr))
}
Some(@SE(NormalTT(expandfun, exp_span))) => {
Some(&NormalTT(expandfun, exp_span)) => {
fld.cx.bt_push(ExpnInfo {
call_site: e.span,
callee: NameAndSpan {
@@ -221,8 +221,8 @@ pub fn expand_mod_items(module_: &ast::_mod, fld: &mut MacroExpander) -> ast::_m
item.attrs.rev_iter().fold(~[*item], |items, attr| {
let mname = attr.name();

match (*fld.extsbox).find(&intern(mname)) {
Some(@SE(ItemDecorator(dec_fn))) => {
match fld.extsbox.find(&intern(mname)) {
Some(&ItemDecorator(dec_fn)) => {
fld.cx.bt_push(ExpnInfo {
call_site: attr.span,
callee: NameAndSpan {
@@ -249,19 +249,14 @@ pub fn expand_mod_items(module_: &ast::_mod, fld: &mut MacroExpander) -> ast::_m
// eval $e with a new exts frame:
macro_rules! with_exts_frame (
($extsboxexpr:expr,$macros_escape:expr,$e:expr) =>
({let extsbox = $extsboxexpr;
let oldexts = *extsbox;
*extsbox = oldexts.push_frame();
extsbox.insert(intern(special_block_name),
@BlockInfo(BlockInfo{macros_escape:$macros_escape,pending_renames:@mut ~[]}));
({$extsboxexpr.push_frame();
$extsboxexpr.info().macros_escape = $macros_escape;
let result = $e;
*extsbox = oldexts;
$extsboxexpr.pop_frame();
result
})
)

static special_block_name : &'static str = " block";

// When we enter a module, record it, for the sake of `module!`
pub fn expand_item(it: @ast::item, fld: &mut MacroExpander)
-> SmallVector<@ast::item> {
@@ -302,11 +297,11 @@ pub fn expand_item_mac(it: @ast::item, fld: &mut MacroExpander)
let extname = &pth.segments[0].identifier;
let extnamestr = ident_to_str(extname);
let fm = fresh_mark();
let expanded = match (*fld.extsbox).find(&extname.name) {
let expanded = match fld.extsbox.find(&extname.name) {
None => fld.cx.span_fatal(pth.span,
format!("macro undefined: '{}!'", extnamestr)),

Some(@SE(NormalTT(expander, span))) => {
Some(&NormalTT(expander, span)) => {
if it.ident.name != parse::token::special_idents::invalid.name {
fld.cx.span_fatal(pth.span,
format!("macro {}! expects no ident argument, \
@@ -326,7 +321,7 @@ pub fn expand_item_mac(it: @ast::item, fld: &mut MacroExpander)
let marked_ctxt = new_mark(fm,ctxt);
expander.expand(fld.cx, it.span, marked_before, marked_ctxt)
}
Some(@SE(IdentTT(expander, span))) => {
Some(&IdentTT(expander, span)) => {
if it.ident.name == parse::token::special_idents::invalid.name {
fld.cx.span_fatal(pth.span,
format!("macro {}! expects an ident argument",
@@ -369,31 +364,14 @@ pub fn expand_item_mac(it: @ast::item, fld: &mut MacroExpander)
MRDef(ref mdef) => {
// yikes... no idea how to apply the mark to this. I'm afraid
// we're going to have to wait-and-see on this one.
insert_macro(*fld.extsbox,intern(mdef.name), @SE((*mdef).ext));
fld.extsbox.insert(intern(mdef.name), (*mdef).ext);
SmallVector::zero()
}
};
fld.cx.bt_pop();
return items;
}


// insert a macro into the innermost frame that doesn't have the
// macro_escape tag.
fn insert_macro(exts: SyntaxEnv, name: ast::Name, transformer: @Transformer) {
let is_non_escaping_block =
|t : &@Transformer| -> bool{
match t {
&@BlockInfo(BlockInfo {macros_escape:false,..}) => true,
&@BlockInfo(BlockInfo {..}) => false,
_ => fail!("special identifier {:?} was bound to a non-BlockInfo",
special_block_name)
}
};
exts.insert_into_frame(name,transformer,intern(special_block_name),
is_non_escaping_block)
}

// expand a stmt
pub fn expand_stmt(s: &Stmt, fld: &mut MacroExpander) -> SmallVector<@Stmt> {
// why the copying here and not in expand_expr?
@@ -414,12 +392,12 @@ pub fn expand_stmt(s: &Stmt, fld: &mut MacroExpander) -> SmallVector<@Stmt> {
}
let extname = &pth.segments[0].identifier;
let extnamestr = ident_to_str(extname);
let fully_expanded: SmallVector<@Stmt> = match (*fld.extsbox).find(&extname.name) {
let fully_expanded: SmallVector<@Stmt> = match fld.extsbox.find(&extname.name) {
None => {
fld.cx.span_fatal(pth.span, format!("macro undefined: '{}'", extnamestr))
}

Some(@SE(NormalTT(expandfun, exp_span))) => {
Some(&NormalTT(expandfun, exp_span)) => {
fld.cx.bt_push(ExpnInfo {
call_site: s.span,
callee: NameAndSpan {
@@ -497,8 +475,6 @@ fn expand_non_macro_stmt(s: &Stmt, fld: &mut MacroExpander)
span: stmt_span
},
node_id) => {
let block_info = get_block_info(*fld.extsbox);
let pending_renames = block_info.pending_renames;

// take it apart:
let @Local {
@@ -522,12 +498,16 @@ fn expand_non_macro_stmt(s: &Stmt, fld: &mut MacroExpander)
let new_name = fresh_name(ident);
new_pending_renames.push((*ident,new_name));
}
let mut rename_fld = renames_to_fold(new_pending_renames);
// rewrite the pattern using the new names (the old ones
// have already been applied):
let rewritten_pat = rename_fld.fold_pat(expanded_pat);
let rewritten_pat = {
let mut rename_fld = renames_to_fold(new_pending_renames);
// rewrite the pattern using the new names (the old ones
// have already been applied):
rename_fld.fold_pat(expanded_pat)
};
// add them to the existing pending renames:
for pr in new_pending_renames.iter() {pending_renames.push(*pr)}
for pr in new_pending_renames.iter() {
fld.extsbox.info().pending_renames.push(*pr)
}
// also, don't forget to expand the init:
let new_init_opt = init.map(|e| fld.fold_expr(e));
let rewritten_local =
@@ -618,16 +598,23 @@ pub fn expand_block(blk: &Block, fld: &mut MacroExpander) -> P<Block> {

// expand the elements of a block.
pub fn expand_block_elts(b: &Block, fld: &mut MacroExpander) -> P<Block> {
let block_info = get_block_info(*fld.extsbox);
let pending_renames = block_info.pending_renames;
let mut rename_fld = renames_to_fold(pending_renames);
let new_view_items = b.view_items.map(|x| fld.fold_view_item(x));
let new_stmts = b.stmts.iter()
.map(|x| rename_fld.fold_stmt(*x)
.expect_one("rename_fold didn't return one value"))
.map(|x| {
let pending_renames = &mut fld.extsbox.info().pending_renames;
let mut rename_fld = renames_to_fold(pending_renames);
rename_fld.fold_stmt(*x).expect_one("rename_fold didn't return one value")
})
.flat_map(|x| fld.fold_stmt(x).move_iter())
.collect();
let new_expr = b.expr.map(|x| fld.fold_expr(rename_fld.fold_expr(x)));
let new_expr = b.expr.map(|x| {
let expr = {
let pending_renames = &mut fld.extsbox.info().pending_renames;
let mut rename_fld = renames_to_fold(pending_renames);
rename_fld.fold_expr(x)
};
fld.fold_expr(expr)
});
P(Block {
view_items: new_view_items,
stmts: new_stmts,
@@ -638,20 +625,11 @@ pub fn expand_block_elts(b: &Block, fld: &mut MacroExpander) -> P<Block> {
})
}

// get the (innermost) BlockInfo from an exts stack
fn get_block_info(exts : SyntaxEnv) -> BlockInfo {
match exts.find_in_topmost_frame(&intern(special_block_name)) {
Some(@BlockInfo(bi)) => bi,
_ => fail!("special identifier {:?} was bound to a non-BlockInfo",
@" block")
}
}

struct IdentRenamer {
renames: @mut ~[(ast::Ident,ast::Name)],
struct IdentRenamer<'a> {
renames: &'a mut RenameList,
}

impl ast_fold for IdentRenamer {
impl<'a> ast_fold for IdentRenamer<'a> {
fn fold_ident(&mut self, id: ast::Ident) -> ast::Ident {
let new_ctxt = self.renames.iter().fold(id.ctxt, |ctxt, &(from, to)| {
new_rename(from, to, ctxt)
@@ -665,7 +643,7 @@ impl ast_fold for IdentRenamer {

// given a mutable list of renames, return a tree-folder that applies those
// renames.
pub fn renames_to_fold(renames: @mut ~[(ast::Ident,ast::Name)]) -> IdentRenamer {
pub fn renames_to_fold<'a>(renames: &'a mut RenameList) -> IdentRenamer<'a> {
IdentRenamer {
renames: renames,
}
@@ -931,7 +909,7 @@ pub fn inject_std_macros(parse_sess: @mut parse::ParseSess,
}

pub struct MacroExpander<'a> {
extsbox: @mut SyntaxEnv,
extsbox: SyntaxEnv,
cx: &'a mut ExtCtxt,
}

@@ -964,15 +942,9 @@ impl<'a> ast_fold for MacroExpander<'a> {
pub fn expand_crate(parse_sess: @mut parse::ParseSess,
cfg: ast::CrateConfig,
c: Crate) -> Crate {
// adding *another* layer of indirection here so that the block
// visitor can swap out one exts table for another for the duration
// of the block. The cleaner alternative would be to thread the
// exts table through the fold, but that would require updating
// every method/element of AstFoldFns in fold.rs.
let extsbox = syntax_expander_table();
let mut cx = ExtCtxt::new(parse_sess, cfg.clone());
let mut expander = MacroExpander {
extsbox: @mut extsbox,
extsbox: syntax_expander_table(),
cx: &mut cx,
};