compile: add line numbers to compiler and create line number table (lnotab)

Author: ncw

compile/compile.go

@@ -68,6 +68,7 @@ const (
 type compiler struct {
 	Code        *py.Code // code being built up
 	Filename    string
+	Lineno      int // current line number
 	OpCodes     Instructions
 	loops       loopstack
 	SymTable    *symtable.SymTable
@@ -144,6 +145,11 @@ func (c *compiler) panicSyntaxErrorf(Ast ast.Ast, format string, a ...interface{
 	panic(err)
 }
 
+// Sets Lineno from an ast node
+func (c *compiler) SetLineno(node ast.Ast) {
+	c.Lineno = node.GetLineno()
+}
+
 // Create a new compiler object at Ast, using private for name mangling
 func (c *compiler) newCompilerScope(compilerScope compilerScopeType, Ast ast.Ast, private string) (newC *compiler) {
 	newSymTable := c.SymTable.FindChild(Ast)
@@ -186,6 +192,7 @@ func (c *compiler) compileAst(Ast ast.Ast, filename string, futureFlags int, don
 	code.Freevars = append(code.Freevars, SymTable.Find(symtable.ScopeFree, symtable.DefFreeClass)...)
 	code.Flags = c.codeFlags(SymTable) | int32(futureFlags&py.CO_COMPILER_FLAGS_MASK)
 	valueOnStack := false
+	c.SetLineno(Ast)
 	switch node := Ast.(type) {
 	case *ast.Module:
 		c.Stmts(c.docString(node.Body, false))
@@ -283,6 +290,7 @@ func (c *compiler) compileAst(Ast ast.Ast, filename string, futureFlags int, don
 	code.Code = c.OpCodes.Assemble()
 	code.Stacksize = int32(c.OpCodes.StackDepth())
 	code.Nlocals = int32(len(code.Varnames))
+	code.Lnotab = string(c.OpCodes.Lnotab())
 	return nil
 }
 
@@ -377,15 +385,19 @@ func (c *compiler) OpArg(Op vm.OpCode, Arg uint32) {
 	if !Op.HAS_ARG() {
 		panic("OpArg called with an instruction which doesn't take an Arg")
 	}
-	c.OpCodes.Add(&OpArg{Op: Op, Arg: Arg})
+	instr := &OpArg{Op: Op, Arg: Arg}
+	instr.SetLineno(c.Lineno)
+	c.OpCodes.Add(instr)
 }
 
 // Compiles an instruction without an argument
 func (c *compiler) Op(op vm.OpCode) {
 	if op.HAS_ARG() {
 		panic("Op called with an instruction which takes an Arg")
 	}
-	c.OpCodes.Add(&Op{Op: op})
+	instr := &Op{Op: op}
+	instr.SetLineno(c.Lineno)
+	c.OpCodes.Add(instr)
 }
 
 // Inserts an existing label
@@ -402,14 +414,17 @@ func (c *compiler) NewLabel() *Label {
 
 // Compiles a jump instruction
 func (c *compiler) Jump(Op vm.OpCode, Dest *Label) {
+	var instr Instruction
 	switch Op {
 	case vm.JUMP_IF_FALSE_OR_POP, vm.JUMP_IF_TRUE_OR_POP, vm.JUMP_ABSOLUTE, vm.POP_JUMP_IF_FALSE, vm.POP_JUMP_IF_TRUE, vm.CONTINUE_LOOP: // Absolute
-		c.OpCodes.Add(&JumpAbs{OpArg: OpArg{Op: Op}, Dest: Dest})
+		instr = &JumpAbs{OpArg: OpArg{Op: Op}, Dest: Dest}
 	case vm.JUMP_FORWARD, vm.SETUP_WITH, vm.FOR_ITER, vm.SETUP_LOOP, vm.SETUP_EXCEPT, vm.SETUP_FINALLY:
-		c.OpCodes.Add(&JumpRel{OpArg: OpArg{Op: Op}, Dest: Dest})
+		instr = &JumpRel{OpArg: OpArg{Op: Op}, Dest: Dest}
 	default:
 		panic("Jump called with non jump instruction")
 	}
+	instr.SetLineno(c.Lineno)
+	c.OpCodes.Add(instr)
 }
 
 /* The test for LOCAL must come before the test for FREE in order to
@@ -963,6 +978,7 @@ func (c *compiler) Stmts(stmts []ast.Stmt) {
 
 // Compile statement
 func (c *compiler) Stmt(stmt ast.Stmt) {
+	c.SetLineno(stmt)
 	switch node := stmt.(type) {
 	case *ast.FunctionDef:
 		// Name          Identifier
@@ -1578,6 +1594,7 @@ func (c *compiler) Exprs(exprs []ast.Expr) {
 
 // Compile and expression
 func (c *compiler) Expr(expr ast.Expr) {
+	c.SetLineno(expr)
 	switch node := expr.(type) {
 	case *ast.BoolOp:
 		// Op     BoolOpNumber

compile/compile_test.go

@@ -19,6 +19,39 @@ func EqString(t *testing.T, name string, a, b string) {
 	}
 }
 
+// Compact the lnotab as python3 seems to generate inefficient ones
+// with spurious zero line number increments.
+func lnotabCompact(t *testing.T, pxs *[]byte) {
+	xs := *pxs
+	newxs := make([]byte, 0, len(xs))
+	carry := 0
+	for i := 0; i < len(xs); i += 2 {
+		d_offset, d_lineno := xs[i], xs[i+1]
+		if d_lineno == 0 {
+			carry += int(d_offset)
+			continue
+		}
+		// FIXME ignoring d_offset overflow
+		d_offset += byte(carry)
+		carry = 0
+		newxs = append(newxs, byte(d_offset), d_lineno)
+	}
+	// if string(newxs) != string(xs) {
+	// 	t.Logf("Compacted\n% x\n% x", xs, newxs)
+	// }
+	*pxs = newxs
+}
+
+func EqLnotab(t *testing.T, name string, aStr, bStr string) {
+	a := []byte(aStr)
+	b := []byte(bStr)
+	lnotabCompact(t, &a)
+	lnotabCompact(t, &b)
+	if string(a) != string(b) {
+		t.Errorf("%s want % x, got % x", name, a, b)
+	}
+}
+
 func EqInt32(t *testing.T, name string, a, b int32) {
 	if a != b {
 		t.Errorf("%s want %d, got %d", name, a, b)
@@ -105,7 +138,9 @@ func EqCode(t *testing.T, name string, a, b *py.Code) {
 	EqCodeCode(t, name+": Code", a.Code, b.Code)
 	EqString(t, name+": Filename", a.Filename, b.Filename)
 	EqString(t, name+": Name", a.Name, b.Name)
-	// FIXME EqString(t, name+": Lnotab", a.Lnotab, b.Lnotab)
+	// The Lnotabs are mostly the same but not entirely
+	// So it is probably not profitable to test them exactly
+	// EqLnotab(t, name+": Lnotab", a.Lnotab, b.Lnotab)
 
 	// []string
 	EqStrings(t, name+": Names", a.Names, b.Names)

compile/instructions.go

@@ -1,8 +1,6 @@
 package compile
 
-import (
-	"github.com/ncw/gpython/vm"
-)
+import "github.com/ncw/gpython/vm"
 
 // FIXME detect if label is not in the instruction stream by setting
 // Pos to 0xFFFF say by default, ie we made a label but forgot to add
@@ -300,6 +298,8 @@ func (is Instructions) StackDepth() int {
 type Instruction interface {
 	Pos() uint32
 	Number() int
+	Lineno() int
+	SetLineno(int)
 	SetPos(int, uint32) bool
 	Size() uint32
 	Output() []byte
@@ -312,8 +312,9 @@ type Resolver interface {
 
 // Position
 type pos struct {
-	n uint32
-	p uint32
+	n      uint32
+	p      uint32
+	lineno int
 }
 
 // Read instruction number
@@ -326,6 +327,16 @@ func (p *pos) Pos() uint32 {
 	return p.p
 }
 
+// Read lineno
+func (p *pos) Lineno() int {
+	return p.lineno
+}
+
+// Set lineno
+func (p *pos) SetLineno(lineno int) {
+	p.lineno = lineno
+}
+
 // Set Position - returns changed
 func (p *pos) SetPos(number int, newPos uint32) bool {
 	p.n = uint32(number)
@@ -439,3 +450,41 @@ func (o *JumpRel) Resolve() {
 		panic("FIXME compile: JUMP_FOWARDS size changed")
 	}
 }
+
+// Creates the lnotab from the instruction stream
+//
+// See Objects/lnotab_notes.txt for the description of the line number table.
+func (is Instructions) Lnotab() []byte {
+	var lnotab []byte
+	old_offset := uint32(0)
+	old_lineno := 1
+	for _, instr := range is {
+		if instr.Size() == 0 {
+			continue
+		}
+		lineno := instr.Lineno()
+		offset := instr.Pos()
+		d_lineno := lineno - old_lineno
+		if d_lineno <= 0 {
+			continue
+		}
+		d_bytecode := offset - old_offset
+		for d_bytecode > 255 {
+			lnotab = append(lnotab, 255, 0)
+			d_bytecode -= 255
+		}
+		for d_lineno > 255 {
+			lnotab = append(lnotab, byte(d_bytecode), 255)
+			d_bytecode = 0
+			d_lineno -= 255
+		}
+		if d_bytecode > 0 {
+			lnotab = append(lnotab, byte(d_bytecode), byte(d_lineno))
+		} else { /* First line of a block; def stmt, etc. */
+			lnotab = append(lnotab, 0, byte(d_lineno))
+		}
+		old_lineno = lineno
+		old_offset = offset
+	}
+	return lnotab
+}

compile/instructions_test.go

@@ -0,0 +1,56 @@
+package compile
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestLnotab(t *testing.T) {
+	for i, test := range []struct {
+		instrs Instructions
+		want   []byte
+	}{
+		{
+			instrs: Instructions{},
+			want:   []byte{},
+		},
+		{
+			instrs: Instructions{
+				&Op{pos: pos{n: 1, p: 10, lineno: 1}},
+				&Op{pos: pos{n: 0, p: 10, lineno: 0}},
+				&Op{pos: pos{n: 1, p: 102, lineno: 1}},
+			},
+			want: []byte{},
+		},
+		{
+			instrs: Instructions{
+				&Op{pos: pos{n: 1, p: 0, lineno: 1}},
+				&Op{pos: pos{n: 1, p: 1, lineno: 2}},
+				&Op{pos: pos{n: 1, p: 2, lineno: 3}},
+			},
+			want: []byte{1, 1, 1, 1},
+		},
+		{
+			// Example from lnotab.txt
+			instrs: Instructions{
+				&Op{pos: pos{n: 1, p: 0, lineno: 1}},
+				&Op{pos: pos{n: 1, p: 6, lineno: 2}},
+				&Op{pos: pos{n: 1, p: 50, lineno: 7}},
+				&Op{pos: pos{n: 1, p: 350, lineno: 307}},
+				&Op{pos: pos{n: 1, p: 361, lineno: 308}},
+			},
+			want: []byte{
+				6, 1,
+				44, 5,
+				255, 0,
+				45, 255,
+				0, 45,
+				11, 1},
+		},
+	} {
+		got := test.instrs.Lnotab()
+		if bytes.Compare(test.want, got) != 0 {
+			t.Errorf("%d: want %d got %d", i, test.want, got)
+		}
+	}
+}

notes.txt

@@ -18,6 +18,7 @@ Things to do before release
 ===========================
 
   * Line numbers
+    * frame.Lasti is pointing past the instruction which puts tracebacks out
   * Subclass builtins
   * pygen
 
@@ -39,10 +40,6 @@ Failes on /usr/lib/python3/dist-packages/mpmath/usertools.py
 Limitations & Missing parts
 ===========================
   * string keys only in dictionaries
-  * line numbers missing in SyntaxErrors
-    * now need to show them when printing SyntaxErrors
-  * line numbers missing in Tracebacks
-    * lnotab etc
   * \N{...} escapes not implemented
   * lots of builtins still to implement
   * FIXME eq && ne should throw an error for a type which doesn' have eq implemented