update lips

Lua/lib/lips/Collector.lua

@@ -1,29 +1,24 @@
 local insert = table.insert
 
 local path = string.gsub(..., "[^.]+$", "")
+local Base = require(path.."Base")
 local Token = require(path.."Token")
+local TokenIter = require(path.."TokenIter")
 local Statement = require(path.."Statement")
-local Muncher = require(path.."Muncher")
 
-local arg_types = { -- for instructions
+local Collector = Base:extend()
-    NUM = true,
-    REG = true,
-    VARSYM = true,
-    LABELSYM = true,
-    RELLABELSYM = true,
-}
-
-local Collector = Muncher:extend()
 function Collector:init(options)
     self.options = options or {}
 end
 
 function Collector:statement(...)
-    local s = Statement(self.fn, self.line, ...)
+    local I = self.iter
+    local s = Statement(I.fn, I.line, ...)
     return s
 end
 
 function Collector:push_data(datum, size)
+    local I = self.iter
     --[[ pseudo-example:
     Statement{type='!DATA',
         {tt='BYTES', tok={0, 1, 2}},
@@ -33,10 +28,12 @@ function Collector:push_data(datum, size)
     }
     --]]
 
+    -- FIXME: optimize the hell out of this garbage, preferably in the lexer
     -- TODO: consider not scrunching data statements, just their tokens
+    -- TODO: concatenate strings; use !BIN instead of !DATA
 
     if type(datum) == 'number' then
-        datum = self:token(datum)
+        datum = I:token(datum)
     end
 
     local last_statement = self.statements[#self.statements]
@@ -58,13 +55,13 @@ function Collector:push_data(datum, size)
             insert(s, datum)
             return
         else
-            self:error('labels are too large to be used in this directive')
+            I:error('labels are too large to be used in this directive')
         end
     elseif datum.tt == 'VARSYM' then
         insert(s, datum:set('size', size))
         return
     elseif datum.tt ~= 'NUM' then
-        self:error('unsupported data type', datum.tt)
+        I:error('unsupported data type', datum.tt)
     end
 
     local sizes = size..'S'
@@ -74,74 +71,67 @@ function Collector:push_data(datum, size)
     if last_token and last_token.tt == sizes then
         t = last_token
     else
-        t = self:token(sizes, {})
+        t = I:token(sizes, {})
         insert(s, t)
         s:validate()
     end
     insert(t.tok, datum.tok)
 end
 
-function Collector:variable()
+function Collector:directive(name)
-    local t = self.t
+    local I = self.iter
-    local t2 = self:advance()
-
-    local s = self:statement('!VAR', t, t2)
-    insert(self.statements, s)
-    self:advance()
-end
-
-function Collector:directive()
-    local name = self.tok
-    self:advance()
     local function add(kind, ...)
         insert(self.statements, self:statement('!'..kind, ...))
     end
+
     if name == 'ORG' or name == 'BASE' then
-        add(name, self:const(nil, 'no labels'))
+        add(name, I:const(nil, 'no labels'))
     elseif name == 'PUSH' or name == 'POP' then
-        add(name, self:const())
+        add(name, I:const())
-        while not self:is_EOL() do
+        while not I:is_EOL() do
-            self:optional_comma()
+            I:eat_comma()
-            add(name, self:const())
+            add(name, I:const())
         end
     elseif name == 'ALIGN' or name == 'SKIP' then
-        if self:is_EOL() and name == 'ALIGN' then
+        if I:is_EOL() and name == 'ALIGN' then
             add(name)
         else
-            local size = self:const(nil, 'no label')
+            local size = I:const(nil, 'no label')
-            if self:is_EOL() then
+            if I:is_EOL() then
                 add(name, size)
             else
-                self:optional_comma()
+                I:eat_comma()
-                add(name, size, self:const(nil, 'no label'))
+                add(name, size, I:const(nil, 'no label'))
             end
         end
+    elseif name == 'BIN' then
+        -- FIXME: not a real directive, just a workaround
+        add(name, I:string())
     elseif name == 'BYTE' or name == 'HALFWORD' or name == 'WORD' then
-        self:push_data(self:const(), name)
+        self:push_data(I:const(), name)
-        while not self:is_EOL() do
+        while not I:is_EOL() do
-            self:optional_comma()
+            I:eat_comma()
-            self:push_data(self:const(), name)
+            self:push_data(I:const(), name)
         end
     elseif name == 'HEX' then
-        if self.tt ~= 'OPEN' then
+        if I.tt ~= 'OPEN' then
-            self:error('expected opening brace for hex directive', self.tt)
+            I:error('expected opening brace for hex directive', I.tt)
         end
-        self:advance()
+        I:next()
 
-        while self.tt ~= 'CLOSE' do
+        while I.tt ~= 'CLOSE' do
-            if self.tt == 'EOL' then
+            if I.tt == 'EOL' then
-                self:advance()
+                I:next()
             else
-                self:push_data(self:const(), 'BYTE')
+                self:push_data(I:const(), 'BYTE')
             end
         end
-        self:advance()
+        I:next()
     elseif name == 'INC' or name == 'INCBIN' then
         -- noop, handled by lexer
-        self:string()
+        I:string()
-        return -- don't expect EOL
     elseif name == 'ASCII' or name == 'ASCIIZ' then
-        local bytes = self:string()
+        local bytes = I:string()
         for i, number in ipairs(bytes.tok) do
             self:push_data(number, 'BYTE')
         end
@@ -149,85 +139,61 @@ function Collector:directive()
             self:push_data(0, 'BYTE')
         end
     elseif name == 'FLOAT' then
-        self:error('unimplemented directive', name)
+        I:error('unimplemented directive', name)
     else
-        self:error('unknown directive', name)
+        I:error('unknown directive', name)
     end
-    self:expect_EOL()
+
+    I:expect_EOL()
 end
 
-function Collector:basic_special()
+function Collector:instruction(name)
-    local name, args = self:special()
+    local I = self.iter
-
+    local s = self:statement(name)
-    local portion
-    if name == 'hi' then
-        portion = 'upperoff'
-    elseif name == 'up' then
-        portion = 'upper'
-    elseif name == 'lo' then
-        portion = 'lower'
-    else
-        self:error('unknown special', name)
-    end
-
-    if #args ~= 1 then
-        self:error(name..' expected one argument', #args)
-    end
-
-    local t = self:token(args[1]):set('portion', portion)
-    return t
-end
-
-function Collector:instruction()
-    local s = self:statement(self.tok)
     insert(self.statements, s)
-    self:advance()
 
-    while self.tt ~= 'EOL' do
+    while I.tt ~= 'EOL' do
-        local t = self.t
+        local t = I.t
-        if self.tt == 'OPEN' then
+        if I.tt == 'OPEN' then
-            t = self:deref()
+            insert(s, I:deref())
-            t.tt = 'DEREF' -- TODO: should just be returned by :deref
+        elseif I.tt == 'UNARY' then
-            insert(s, t)
+            local peek = assert(I:peek())
-        elseif self.tt == 'UNARY' then
-            local peek = self.tokens[self.i + 1]
             if peek.tt == 'VARSYM' then
                 local negate = t.tok == -1
-                t = self:advance()
+                t = I:next()
                 t = Token(t):set('negate', negate)
                 insert(s, t)
-                self:advance()
+                I:next()
             elseif peek.tt == 'EOL' or peek.tt == 'SEP' then
                 local tok = t.tok == 1 and '+' or t.tok == -1 and '-'
-                t = Token(self.fn, self.line, 'RELLABELSYM', tok)
+                t = Token(I.fn, I.line, 'RELLABELSYM', tok)
                 insert(s, t)
-                self:advance()
+                I:next()
             else
-                self:error('unexpected token after unary operator', peek.tt)
+                I:error('unexpected token after unary operator', peek.tt)
             end
-        elseif self.tt == 'SPECIAL' then
+        elseif I.tt == 'SPECIAL' then
-            t = self:basic_special()
+            t = I:basic_special()
             insert(s, t)
-            self:advance()
+            I:next()
-        elseif self.tt == 'SEP' then
+        elseif I.tt == 'SEP' then
-            self:error('extraneous comma')
+            I:error('extraneous comma')
-        elseif not arg_types[self.tt] then
+        elseif not I.arg_types[I.tt] then
-            self:error('unexpected argument type in instruction', self.tt)
+            I:error('unexpected argument type in instruction', I.tt)
         else
             insert(s, t)
-            self:advance()
+            I:next()
         end
-        self:optional_comma()
+        I:eat_comma()
     end
 
-    self:expect_EOL()
+    I:expect_EOL()
     s:validate()
 end
 
 function Collector:collect(tokens, fn)
-    self.tokens = tokens
+    self.iter = TokenIter(tokens)
-    self.fn = fn or '(string)'
+    local I = self.iter
-    self.main_fn = self.fn
 
     self.statements = {}
 
@@ -241,29 +207,27 @@ function Collector:collect(tokens, fn)
         insert(self.statements, s)
     end
 
-    self.i = 0 -- set up Muncher iteration
+    for t in I do
-    self:advance() -- load up the first token
+        if t.tt == 'EOF' then
-    while true do
+            -- noop
-        if self.tt == 'EOF' then
+        elseif t.tt == 'EOL' then
-            -- don't break if this is an included file's EOF
+            -- noop; empty line
-            if self.fn == self.main_fn then
+        elseif t.tt == 'LABEL' or t.tt == 'RELLABEL' then
-                break
+            insert(self.statements, self:statement('!LABEL', t))
-            end
+        elseif t.tt == 'VAR' then
-            self:advance()
+            local t2 = I:next()
-        elseif self.tt == 'EOL' then
+            I:next()
-            -- empty line
+            local s = self:statement('!VAR', t, t2)
-            self:advance()
+            insert(self.statements, s)
-        elseif self.tt == 'VAR' then
+            I:expect_EOL()
-            self:variable() -- handles advancing
+        elseif t.tt == 'DIR' then
-        elseif self.tt == 'LABEL' or self.tt == 'RELLABEL' then
+            I:next()
-            insert(self.statements, self:statement('!LABEL', self.t))
+            self:directive(t.tok)
-            self:advance()
+        elseif t.tt == 'INSTR' then
-        elseif self.tt == 'DIR' then
+            I:next()
-            self:directive() -- handles advancing
+            self:instruction(t.tok)
-        elseif self.tt == 'INSTR' then
-            self:instruction() -- handles advancing
         else
-            self:error('expected starting token for statement', self.tt)
+            I:error('expected starting token for statement', t.tt)
         end
     end
 

Lua/lib/lips/Dumper.lua

@@ -1,3 +1,4 @@
+local byte = string.byte
 local floor = math.floor
 local format = string.format
 local insert = table.insert
@@ -253,6 +254,10 @@ function Dumper:load(statements)
                 s.length = util.measure_data(s) -- cache for next pass
                 self.pos = self.pos + s.length
                 insert(new_statements, s)
+            elseif s.type == '!BIN' then
+                s.length = #s[1].tok
+                self.pos = self.pos + s.length
+                insert(new_statements, s)
             elseif s.type == '!ORG' then
                 self.pos = s[1].tok
                 insert(new_statements, s)
@@ -378,6 +383,9 @@ function Dumper:load(statements)
             end
             self.pos = self.pos + (s.length or util.measure_data(s))
             insert(new_statements, s)
+        elseif s.type == '!BIN' then
+            self.pos = self.pos + s.length
+            insert(new_statements, s)
         elseif s.type == '!ORG' then
             self.pos = s[1].tok
             insert(new_statements, s)
@@ -423,6 +431,12 @@ function Dumper:dump()
                     error('Internal Error: unknown !DATA token')
                 end
             end
+        elseif s.type == '!BIN' then
+            local data = s[1].tok
+            for i=1, #data do
+                self.writer(self.pos, byte(data, i))
+                self.pos = self.pos + 1
+            end
         elseif s.type == '!ORG' then
             self.pos = s[1].tok
         else

Lua/lib/lips/Expander.lua

@@ -0,0 +1,96 @@
+local insert = table.insert
+
+local path = string.gsub(..., "[^.]+$", "")
+local data = require(path.."data")
+local overrides = require(path.."overrides")
+local Statement = require(path.."Statement")
+local Reader = require(path.."Reader")
+
+local Expander = Reader:extend()
+function Expander:init(options)
+    self.options = options or {}
+end
+
+function Expander:statement(...)
+    local s = Statement(self.fn, self.line, ...)
+    return s
+end
+
+function Expander:push(s)
+    s:validate()
+    insert(self.statements, s)
+end
+
+function Expander:push_new(...)
+    self:push(self:statement(...))
+end
+
+function Expander:pop(kind)
+    local ret
+    if kind == nil then
+        ret = self.s[self.i]
+    elseif kind == 'CPU' then
+        ret = self:register(data.registers)
+    elseif kind == 'DEREF' then
+        ret = self:deref()
+    elseif kind == 'CONST' then
+        ret = self:const()
+    elseif kind == 'END' then
+        if self.s[self.i] ~= nil then
+            self:error('expected EOL; too many arguments')
+        end
+        return -- don't increment self.i past end of arguments
+    else
+        error('Internal Error: unknown kind, got '..tostring(kind))
+    end
+    self.i = self.i + 1
+    return ret
+end
+
+function Expander:expand(statements)
+    -- fourth pass: expand pseudo-instructions and register arguments
+    self.statements = {}
+    for i, s in ipairs(statements) do
+        self.s = s
+        self.fn = s.fn
+        self.line = s.line
+        if s.type:sub(1, 1) == '!' then
+            self:push(s)
+        else
+            local name = s.type
+            local h = data.instructions[name]
+            if h == nil then
+                error('Internal Error: unknown instruction')
+            end
+
+            if data.one_register_variants[name] then
+                self.i = 1
+                local a = self:register(data.all_registers)
+                local b = s[2]
+                if b == nil or b.tt ~= 'REG' then
+                    insert(s, 2, self:token(a))
+                end
+            elseif data.two_register_variants[name] then
+                self.i = 1
+                local a = self:register(data.all_registers)
+                local b = self:register(data.all_registers)
+                local c = s[3]
+                if c == nil or c.tt ~= 'REG' then
+                    insert(s, 2, self:token(a))
+                end
+            end
+
+            if overrides[name] then
+                self.i = 1
+                overrides[name](self, name)
+                self:pop('END')
+            else
+                self:push(s)
+            end
+        end
+    end
+
+    return self.statements
+end
+
+return Expander

Lua/lib/lips/Lexer.lua

@@ -289,7 +289,7 @@ function Lexer:lex_string_naive(yield) -- no escape sequences
     yield('STRING', buff)
 end
 
-function Lexer:lex_include(_yield)
+function Lexer:lex_filename(_yield)
     self:read_spaces()
     local fn
     self:lex_string_naive(function(tt, tok)
@@ -297,6 +297,18 @@ function Lexer:lex_include(_yield)
     end)
     _yield('STRING', fn, self.fn, self.line)
 
+    if self.chr ~= '\n' then
+        self:error('expected EOL after filename')
+    end
+    _yield('EOL', '\n', self.fn, self.line)
+    self:nextc()
+
+    return fn
+end
+
+function Lexer:lex_include(_yield)
+    local fn = self:lex_filename(_yield)
+
     if self.options.path then
         fn = self.options.path..fn
     end
@@ -308,25 +320,15 @@ function Lexer:lex_include(_yield)
 end
 
 function Lexer:lex_include_binary(_yield)
-    self:read_spaces()
+    local fn = self:lex_filename(_yield)
-    local fn
-    self:lex_string_naive(function(tt, tok)
-        fn = tok
-    end)
-    _yield('STRING', fn, self.fn, self.line)
 
     -- TODO: allow optional offset and size arguments
     if self.options.path then
         fn = self.options.path..fn
     end
     local data = util.readfile(fn, true)
-
+    _yield('DIR', 'BIN', fn, 0)
-    -- FIXME: this allocates a table for each byte.
+    _yield('STRING', data, fn, 0)
-    --        this could easily cause performance issues on big files.
-    _yield('DIR', 'BYTE', fn, 0)
-    for b in string.gfind(data, '.') do
-        _yield('NUM', string.byte(b), fn, 0)
-    end
 end
 
 function Lexer:lex_expression(yield)

Lua/lib/lips/Parser.lua

@@ -6,6 +6,7 @@ local Token = require(path.."Token")
 local Lexer = require(path.."Lexer")
 local Collector = require(path.."Collector")
 local Preproc = require(path.."Preproc")
+local Expander = require(path.."Expander")
 local Dumper = require(path.."Dumper")
 
 local Parser = Base:extend()
@@ -37,53 +38,28 @@ function Parser:tokenize(asm)
     assert(#tokens > 0, 'Internal Error: no tokens after preprocessing')
 
     local collector = Collector(self.options)
-    self.statements = collector:collect(tokens, self.main_fn)
+    return collector:collect(tokens, self.main_fn)
 end
 
-function Parser:debug_dump()
+function Parser:dump()
-    local boring = {
-        tt = true,
-        tok = true,
-        fn = true,
-        line = true,
-    }
     for i, s in ipairs(self.statements) do
-        local values = ''
+        print(s.line, s.type, s:dump())
-        for j, t in ipairs(s) do
-            local tok = t.tok
-            if type(tok) == 'number' then
-                tok = ("$%X"):format(tok)
-            end
-            values = values..'\t'..t.tt..'('..tostring(tok)..')'
-            for k, v in pairs(t) do
-                if not boring[k] then
-                    values = values..'['..k..'='..tostring(v)..']'
-                end
-            end
-        end
-        values = values:sub(2)
-        print(s.line, s.type, values)
     end
 end
 
 function Parser:parse(asm)
-    self:tokenize(asm)
+    self.statements = self:tokenize(asm)
+    if self.options.debug_token then self:dump() end
 
-    if self.options.debug_token then self:debug_dump() end
+    self.statements = Preproc(self.options):process(self.statements)
+    if self.options.debug_pre then self:dump() end
 
-    local preproc = Preproc(self.options)
+    self.statements = Expander(self.options):expand(self.statements)
-    self.statements = preproc:process(self.statements)
+    if self.options.debug_post then self:dump() end
-
-    if self.options.debug_pre then self:debug_dump() end
-
-    self.statements = preproc:expand(self.statements)
-
-    if self.options.debug_post then self:debug_dump() end
 
     local dumper = Dumper(self.writer, self.options)
     self.statements = dumper:load(self.statements)
-
+    if self.options.debug_asm then self:dump() end
-    if self.options.debug_asm then self:debug_dump() end
 
     if self.options.labels then
         dumper:export_labels(self.options.labels)

Lua/lib/lips/Preproc.lua

@@ -1,31 +1,33 @@
+local abs = math.abs
 local insert = table.insert
 
 local path = string.gsub(..., "[^.]+$", "")
-local data = require(path.."data")
+local Base = require(path.."Base")
-local overrides = require(path.."overrides")
-local Statement = require(path.."Statement")
-local Reader = require(path.."Reader")
 local Expression = require(path.."Expression")
+local util = require(path.."util")
 
-local abs = math.abs
+local signs = util.signs
 
-local function signs(s)
+local Preproc = Base:extend()
-    local start, end_ = s:find('[+-]+')
-    if start ~= 1 then
-        return 0
-    end
-    if s:sub(1, 1) == '+' then
-        return end_
-    elseif s:sub(1, 1) == '-' then
-        return -end_
-    end
-end
-
-local Preproc = Reader:extend()
 function Preproc:init(options)
     self.options = options or {}
 end
 
+function Preproc:iter(statements)
+    assert(statements)
+    local i = 0
+    return function()
+        i = i + 1
+        local s = statements[i]
+        if s == nil then return end
+        self.i = i
+        self.s = s
+        self.fn = s.fn
+        self.line = s.line
+        return s
+    end
+end
+
 function Preproc:lookup(t)
     if t.tt == 'VARSYM' then
         local name = t.tok
@@ -110,8 +112,6 @@ function Preproc:check(s, i, tt)
 end
 
 function Preproc:process(statements)
-    self.statements = statements
-
     self.variables = {}
     self.plus_labels = {} -- constructed forwards
     self.minus_labels = {} -- constructed backwards
@@ -119,39 +119,29 @@ function Preproc:process(statements)
 
     -- first pass: resolve variables and collect relative labels
     local new_statements = {}
-    for i=1, #self.statements do
+    for s in self:iter(statements) do
-        local s = self.statements[i]
+        -- directive, label, etc.
-        self.fn = s.fn
+        if s.type == '!VAR' then
-        self.line = s.line
+            local a = self:check(s, 1, 'VAR')
-        if s.type:sub(1, 1) == '!' then
+            local b = self:check(s, 2, 'NUM')
-            -- directive, label, etc.
+            self.variables[a] = b
-            if s.type == '!VAR' then
+        elseif s.type == '!LABEL' then
-                local a = self:check(s, 1, 'VAR')
+            if s[1].tt == 'RELLABEL' then
-                local b = self:check(s, 2, 'NUM')
+                local label = s[1].tok
-                self.variables[a] = b
+                local rl = {
-            elseif s.type == '!LABEL' then
+                    index = #new_statements + 1,
-                if s[1].tt == 'RELLABEL' then
+                    name = label:sub(2)
-                    local label = s[1].tok
+                }
-                    local rl = {
+                local c = label:sub(1, 1)
-                        index = #new_statements + 1,
+                if c == '+' then
-                        name = label:sub(2)
+                    insert(self.plus_labels, rl)
-                    }
+                elseif c == '-' then
-                    local c = label:sub(1, 1)
+                    insert(self.minus_labels, 1, rl) -- remember, it's backwards
-                    if c == '+' then
+                else
-                        insert(self.plus_labels, rl)
+                    error('Internal Error: unexpected token for relative label')
-                    elseif c == '-' then
-                        insert(self.minus_labels, 1, rl) -- remember, it's backwards
-                    else
-                        error('Internal Error: unexpected token for relative label')
-                    end
                 end
-                insert(new_statements, s)
-            else
-                for j, t in ipairs(s) do
-                    self:lookup(t)
-                end
-                insert(new_statements, s)
             end
+            insert(new_statements, s)
         else
             -- regular instruction
             for j, t in ipairs(s) do
@@ -163,21 +153,14 @@ function Preproc:process(statements)
 
     -- second pass: resolve relative labels
     self.do_labels = true
-    for i=1, #new_statements do
+    for s in self:iter(new_statements) do
-        self.i = i -- make visible to :lookup
-        local s = new_statements[i]
-        self.fn = s.fn
-        self.line = s.line
         for j, t in ipairs(s) do
             self:lookup(t)
         end
     end
 
     -- third pass: evaluate constant expressions
-    for i=1, #new_statements do
+    for s in self:iter(new_statements) do
-        local s = new_statements[i]
-        self.fn = s.fn
-        self.line = s.line
         for j, t in ipairs(s) do
             if t.tt == 'EXPR' then
                 local expr = Expression()
@@ -194,89 +177,4 @@ function Preproc:process(statements)
     return new_statements
 end
 
-function Preproc:statement(...)
-    self.fn = self.s.fn
-    self.line = self.s.line
-    local s = Statement(self.fn, self.line, ...)
-    return s
-end
-
-function Preproc:push(s)
-    s:validate()
-    insert(self.statements, s)
-end
-
-function Preproc:push_new(...)
-    self:push(self:statement(...))
-end
-
-function Preproc:pop(kind)
-    local ret
-    if kind == nil then
-        ret = self.s[self.i]
-    elseif kind == 'CPU' then
-        ret = self:register(data.registers)
-    elseif kind == 'DEREF' then
-        ret = self:deref()
-    elseif kind == 'CONST' then
-        ret = self:const()
-    elseif kind == 'END' then
-        if self.s[self.i] ~= nil then
-            self:error('expected EOL; too many arguments')
-        end
-        return -- don't increment self.i past end of arguments
-    else
-        error('Internal Error: unknown kind, got '..tostring(kind))
-    end
-    self.i = self.i + 1
-    return ret
-end
-
-function Preproc:expand(statements)
-    -- fourth pass: expand pseudo-instructions and register arguments
-    self.statements = {}
-    for i=1, #statements do
-        local s = statements[i]
-        self.s = s
-        self.fn = s.fn
-        self.line = s.line
-        if s.type:sub(1, 1) == '!' then
-            self:push(s)
-        else
-            local name = s.type
-            local h = data.instructions[name]
-            if h == nil then
-                error('Internal Error: unknown instruction')
-            end
-
-            if data.one_register_variants[name] then
-                self.i = 1
-                local a = self:register(data.all_registers)
-                local b = s[2]
-                if b == nil or b.tt ~= 'REG' then
-                    insert(s, 2, self:token(a))
-                end
-            elseif data.two_register_variants[name] then
-                self.i = 1
-                local a = self:register(data.all_registers)
-                local b = self:register(data.all_registers)
-                local c = s[3]
-                if c == nil or c.tt ~= 'REG' then
-                    insert(s, 2, self:token(a))
-                end
-            end
-
-            if overrides[name] then
-                self.i = 1
-                overrides[name](self, name)
-                self:pop('END')
-            else
-                self:push(s)
-            end
-        end
-    end
-
-    return self.statements
-end
-
 return Preproc

Lua/lib/lips/Statement.lua

@@ -50,4 +50,28 @@ function Statement:validate(n)
     end
 end
 
+local boring = {
+    tt = true,
+    tok = true,
+    fn = true,
+    line = true,
+}
+
+function Statement:dump()
+    local values = ''
+    for j, t in ipairs(self) do
+        local tok = t.tok
+        if type(tok) == 'number' then
+            tok = ("$%X"):format(tok)
+        end
+        values = values..'\t'..t.tt..'('..tostring(tok)..')'
+        for k, v in pairs(t) do
+            if not boring[k] then
+                values = values..'['..k..'='..tostring(v)..']'
+            end
+        end
+    end
+    return values:sub(2)
+end
+
 return Statement

Lua/lib/lips/TokenIter.lua

@@ -2,11 +2,21 @@ local format = string.format
 local insert = table.insert
 
 local path = string.gsub(..., "[^.]+$", "")
-local data = require(path.."data")
-local Base = require(path.."Base")
 local Token = require(path.."Token")
 
-local arg_types = {
+local Iter = {}
+function Iter:__call()
+    return self:next(1)
+end
+
+local TokenIter = {}
+function TokenIter:init(tokens)
+    assert(tokens ~= nil)
+    self.tokens = tokens
+    self:reset()
+end
+
+TokenIter.arg_types = {
     NUM = true,
     REG = true,
     VARSYM = true,
@@ -14,17 +24,57 @@ local arg_types = {
     RELLABELSYM = true,
 }
 
-local Muncher = Base:extend()
+function TokenIter:error(msg, got)
--- no base init method
-
-function Muncher:error(msg, got)
     if got ~= nil then
         msg = msg..', got '..tostring(got)
     end
     error(format('%s:%d: Error: %s', self.fn, self.line, msg), 2)
 end
 
-function Muncher:token(t, val)
+function TokenIter:reset()
+    self.i = 0
+    self.tt = nil
+    self.tok = nil
+    self.fn = nil
+    self.line = nil
+    self.ended = false
+end
+
+function TokenIter:advance(n)
+    n = n or 0
+    if self.ended then
+        error('Internal Error: attempted to advance iterator past end', 2 + n)
+    end
+
+    self.i = self.i + 1
+    self.t = self.tokens[self.i]
+    if self.t == nil then
+        self.tt = nil
+        self.tok = nil
+        self.fn = nil
+        self.line = nil
+        self.ended = true
+    else
+        self.tt = self.t.tt
+        self.tok = self.t.tok
+        self.fn = self.t.fn
+        self.line = self.t.line
+    end
+end
+
+function TokenIter:next(n)
+    n = n or 0
+    self:advance(n + 1)
+    if self.t then return self.t end
+end
+
+function TokenIter:peek()
+    return self.tokens[self.i + 1]
+end
+
+-- now begins the parsing stuff
+
+function TokenIter:token(t, val)
     -- note: call Token directly if you want to specify fn and line manually
     if type(t) == 'table' then
         t.fn = self.fn
@@ -37,36 +87,25 @@ function Muncher:token(t, val)
     end
 end
 
-function Muncher:advance()
+function TokenIter:is_EOL()
-    self.i = self.i + 1
-    self.t = self.tokens[self.i]
-    self.tt = self.t.tt
-    self.tok = self.t.tok
-    self.fn = self.t.fn
-    self.line = self.t.line
-    return self.t
-end
-
-function Muncher:is_EOL()
     return self.tt == 'EOL' or self.tt == 'EOF'
 end
 
-function Muncher:expect_EOL()
+function TokenIter:expect_EOL()
     if self:is_EOL() then
-        self:advance()
         return
     end
     self:error('expected end of line', self.tt)
 end
 
-function Muncher:optional_comma()
+function TokenIter:eat_comma()
     if self.tt == 'SEP' and self.tok == ',' then
         self:advance()
         return true
     end
 end
 
-function Muncher:number()
+function TokenIter:number()
     if self.tt ~= 'NUM' then
         self:error('expected number', self.tt)
     end
@@ -75,7 +114,7 @@ function Muncher:number()
     return self:token(t)
 end
 
-function Muncher:string()
+function TokenIter:string()
     if self.tt ~= 'STRING' then
         self:error('expected string', self.tt)
     end
@@ -84,7 +123,7 @@ function Muncher:string()
     return self:token(t)
 end
 
-function Muncher:register(registers)
+function TokenIter:register(registers)
     registers = registers or data.registers
     if self.tt ~= 'REG' then
         self:error('expected register', self.tt)
@@ -97,7 +136,7 @@ function Muncher:register(registers)
     return self:token(t)
 end
 
-function Muncher:deref()
+function TokenIter:deref()
     if self.tt ~= 'OPEN' then
         self:error('expected opening parenthesis for dereferencing', self.tt)
     end
@@ -111,10 +150,10 @@ function Muncher:deref()
         self:error('expected closing parenthesis for dereferencing', self.tt)
     end
     self:advance()
-    return self:token(t)
+    return self:token(t):set('tt', 'DEREF')
 end
 
-function Muncher:const(relative, no_label)
+function TokenIter:const(relative, no_label)
     local good = {
         NUM = true,
         EXPR = true,
@@ -132,7 +171,7 @@ function Muncher:const(relative, no_label)
     return t
 end
 
-function Muncher:special()
+function TokenIter:special()
     if self.tt ~= 'SPECIAL' then
         self:error('expected special name to call', self.tt)
     end
@@ -145,7 +184,7 @@ function Muncher:special()
     local args = {}
     while true do
         local arg = self:advance()
-        if not arg_types[arg.tt] then
+        if not self.arg_types[arg.tt] then
             self:error('invalid argument type', arg.tt)
         else
             self:advance()
@@ -163,4 +202,39 @@ function Muncher:special()
     return name, args
 end
 
-return Muncher
+function TokenIter:basic_special()
+    local name, args = self:special()
+
+    local portion
+    if name == 'hi' then
+        portion = 'upperoff'
+    elseif name == 'up' then
+        portion = 'upper'
+    elseif name == 'lo' then
+        portion = 'lower'
+    else
+        self:error('unknown special', name)
+    end
+
+    if #args ~= 1 then
+        self:error(name..' expected one argument', #args)
+    end
+
+    local t = self:token(args[1]):set('portion', portion)
+    return t
+end
+
+-- TODO: move this boilerplate elsewhere
+
+local MetaBlah = {
+    __index = TokenIter,
+    __call = TokenIter.next,
+}
+
+local ClassBlah = {}
+function ClassBlah:__call(...)
+    local obj = setmetatable({}, MetaBlah)
+    return obj, obj:init(...)
+end
+
+return setmetatable(TokenIter, ClassBlah)

Lua/lib/lips/overrides.lua

@@ -47,7 +47,7 @@ local function li(self, buffer, dest, im)
 end
 
 local overrides = {}
--- note: "self" is an instance of Preproc
+-- note: "self" is an instance of Expander
 
 local function tob_override(self, name)
     -- handle all the addressing modes for lw/sw-like instructions

Lua/lib/lips/util.lua

@@ -14,7 +14,7 @@ local function readfile(fn, binary)
 end
 
 local function bitrange(x, lower, upper)
-    return floor(x/2^lower) % 2^(upper - lower + 1)
+    return floor(x / 2^lower) % 2^(upper - lower + 1)
 end
 
 local function parent(t)
@@ -25,6 +25,18 @@ local function parent(t)
     return mt.__index
 end
 
+local function signs(s)
+    local start, end_ = s:find('[+-]+')
+    if start ~= 1 then
+        return 0
+    end
+    if s:sub(1, 1) == '+' then
+        return end_
+    elseif s:sub(1, 1) == '-' then
+        return -end_
+    end
+end
+
 -- http://stackoverflow.com/a/9279009
 local loadcode
 if setfenv and loadstring then -- 5.1, JIT
@@ -77,6 +89,7 @@ return {
     readfile = readfile,
     bitrange = bitrange,
     parent = parent,
+    signs = signs,
     loadcode = loadcode,
     measure_data = measure_data,
 }