update lips

Lua/lib/lips/Dumper.lua

@@ -12,7 +12,7 @@ function Dumper:init(writer, fn, options)
     self.writer = writer
     self.fn = fn or '(string)'
     self.options = options or {}
-    self.labels = {}
+    self.labels = setmetatable({}, {__index=options.labels})
     self.commands = {}
     self.pos = options.offset or 0
     self.lastcommand = nil
@@ -22,6 +22,17 @@ function Dumper:error(msg)
     error(format('%s:%d: Error: %s', self.fn, self.line, msg), 2)
 end
 
+function Dumper:export_labels(t)
+    for k, v in pairs(self.labels) do
+        -- only return valid labels; those that don't begin with a number
+        -- (relative labels are invalid)
+        if not tostring(k):sub(1, 1):find('%d') then
+            t[k] = v
+        end
+    end
+    return t
+end
+
 function Dumper:advance(by)
     self.pos = self.pos + by
 end
@@ -138,21 +149,25 @@ function Dumper:desym(t)
         end
         return rel % 0x10000
     elseif type(t.tok) == 'number' then
+        if t.offset then
+            return t.tok + t.offset
+        end
         return t.tok
     elseif t.tt == 'REG' then
         assert(data.all_registers[t.tok], 'Internal Error: unknown register')
         return data.registers[t.tok] or data.fpu_registers[t.tok] or data.sys_registers[t.tok]
-    elseif t.tt == 'LABELSYM' then
+    elseif t.tt == 'LABELSYM' or t.tt == 'LABELREL' then
         local label = self.labels[t.tok]
         if label == nil then
             self:error('undefined label')
         end
+        if t.offset then
+            label = label + t.offset
+        end
+        if t.tt == 'LABELSYM' then
             return label
-    elseif t.tt == 'LABELREL' then
-        local label = self.labels[t.tok]
-        if label == nil then
-            self:error('undefined label')
         end
+
         label = label % 0x80000000
         local pos = self.pos % 0x80000000
         local rel = floor(label/4) - 1 - floor(pos/4)

Lua/lib/lips/Lexer.lua

@@ -89,6 +89,10 @@ function Lexer:read_chars(pattern)
     return buff
 end
 
+function Lexer:read_spaces()
+    return self:read_chars('[ \t]')
+end
+
 function Lexer:read_decimal()
     local buff = self:read_chars('%d')
     local num = tonumber(buff)
@@ -262,7 +266,7 @@ function Lexer:lex_string_naive(yield) -- no escape sequences
 end
 
 function Lexer:lex_include(_yield)
-    self:read_chars('%s')
+    self:read_spaces()
     local fn
     self:lex_string_naive(function(tt, tok)
         fn = tok
@@ -274,6 +278,24 @@ function Lexer:lex_include(_yield)
     sublexer:lex(_yield)
 end
 
+function Lexer:lex_include_binary(_yield)
+    self:read_spaces()
+    local fn
+    self:lex_string_naive(function(tt, tok)
+        fn = tok
+    end)
+    if self.options.path then
+        fn = self.options.path..fn
+    end
+    -- NOTE: this allocates two tables for each byte.
+    --       this could easily cause performance issues on big files.
+    local data = util.readfile(fn, true)
+    for b in string.gfind(data, '.') do
+        _yield('DIR', 'BYTE', fn, 0)
+        _yield('NUM', string.byte(b), fn, 0)
+    end
+end
+
 function Lexer:lex(_yield)
     local function yield(tt, tok)
         return _yield(tt, tok, self.fn, self.line)
@@ -334,6 +356,9 @@ function Lexer:lex(_yield)
             if up == 'INC' or up == 'INCASM' or up == 'INCLUDE' then
                 yield('DIR', 'INC')
                 self:lex_include(_yield)
+            elseif up == 'INCBIN' then
+                yield('DIR', 'INCBIN')
+                self:lex_include_binary(_yield)
             else
                 yield('DIR', up)
             end
@@ -371,16 +396,25 @@ function Lexer:lex(_yield)
         elseif self.chr == '+' or self.chr == '-' then
             local sign_chr = self.chr
             local sign = sign_chr == '+' and 1 or -1
-            local buff = self:read_chars('%'..self.chr)
+            local signs = self:read_chars('%'..self.chr)
-            if #buff == 1 and self.chr == ':' then
+            local name = ''
+            if self.chr:find('[%a_]') then
+                name = self:read_chars('[%w_]')
+            end
+            if #signs == 1 and self.chr == ':' then
                 self:nextc()
-                yield('RELLABEL', sign_chr)
+                yield('RELLABEL', signs..name)
             else
+                self:read_spaces()
                 local n = self:read_number()
                 if n then
                     yield('NUM', sign*n)
+                elseif #signs == 1 and name == '' then
+                    -- this could be a RELLABELSYM
+                    -- we'll have to let the preproc figure it out
+                    yield('UNARY', sign)
                 else
-                    yield('RELLABELSYM', sign*#buff)
+                    yield('RELLABELSYM', signs..name)
                 end
             end
         else

Lua/lib/lips/Parser.lua

@@ -54,7 +54,7 @@ function Parser:directive()
             add(name, self:const().tok)
         end
         self:expect_EOL()
-    elseif name == 'INC' then
+    elseif name == 'INC' or name == 'INCBIN' then
         -- noop, handled by lexer
     elseif name == 'ASCII' or name == 'ASCIIZ' then
         local bytes = self:string()
@@ -65,8 +65,6 @@ function Parser:directive()
             add('BYTE', 0)
         end
         self:expect_EOL()
-    elseif name == 'INCBIN' then
-        self:error('unimplemented')
     elseif name == 'FLOAT' then
         self:error('unimplemented')
     else
@@ -179,6 +177,7 @@ function Parser:instruction()
     elseif overrides[name] then
         overrides[name](self, name)
     elseif h[2] == 'tob' then -- TODO: or h[2] == 'Tob' then
+        -- handle all the addressing modes for lw/sw-like instructions
         local lui = data.instructions['LUI']
         local addu = data.instructions['ADDU']
         local args = {}
@@ -191,6 +190,9 @@ function Parser:instruction()
             local lui_args = {}
             local addu_args = {}
             local o = self:const()
+            if self.tt == 'NUM' then
+                o:set('offset', self:const().tok)
+            end
             args.offset = self:token(o)
             if not o.portion then
                 args.offset:set('portion', 'lower')
@@ -274,6 +276,9 @@ function Parser:parse(asm)
             self:error('unexpected token (unknown instruction?)')
         end
     end
+    if self.options.labels then
+        self.dumper:export_labels(self.options.labels)
+    end
     return self.dumper:dump()
 end
 

Lua/lib/lips/Preproc.lua

@@ -5,6 +5,27 @@ local util = require "lips.util"
 local Muncher = require "lips.Muncher"
 local Token = require "lips.Token"
 
+local abs = math.abs
+
+local function signs(s)
+    local start, end_ = s:find('[+-]+')
+    if start ~= 1 then
+        return 0
+    end
+    if s:sub(1, 1) == '+' then
+        return end_
+    elseif s:sub(1, 1) == '-' then
+        return -end_
+    end
+end
+
+local function RelativeLabel(index, name)
+    return {
+        index = index,
+        name = name,
+    }
+end
+
 local Preproc = util.Class(Muncher)
 function Preproc:init(options)
     self.options = options or {}
@@ -17,11 +38,26 @@ function Preproc:process(tokens)
     local plus_labels = {} -- constructed forwards
     local minus_labels = {} -- constructed backwards
 
-    -- first pass: resolve defines, collect relative labels
+    -- first pass: resolve unary ops, defines, and collect relative labels
     local new_tokens = {}
     self.i = 0
     while self.i < #self.tokens do
         local t = self:advance()
+        local sign = 1
+        if t.tt == 'UNARY' then
+            sign = t.tok
+            local peek = self.tokens[self.i + 1]
+            if peek.tt == 'UNARY' then
+                self:error('unary operators cannot be chained')
+            elseif peek.tt == 'EOL' or peek.tt == 'SEP' then
+                t.tt = 'RELLABELSYM'
+                t.tok = sign == 1 and '+' or sign == -1 and '-'
+            elseif peek.tt == 'DEFSYM' then
+                t = self:advance()
+            else
+                self:error('expected a symbolic constant after unary operator')
+            end
+        end
         if t.tt == nil then
             error('Internal Error: missing token')
         elseif t.tt == 'DEF' then
@@ -36,12 +72,14 @@ function Preproc:process(tokens)
             if tok == nil then
                 self:error('undefined define') -- uhhh nice wording
             end
-            insert(new_tokens, self:token(tt, tok))
+            insert(new_tokens, self:token(tt, tok * sign))
         elseif t.tt == 'RELLABEL' then
-            if t.tok == '+' then
+            local label = t.tok or ''
-                insert(plus_labels, #new_tokens + 1)
+            local rl = RelativeLabel(#new_tokens + 1, label:sub(2))
-            elseif t.tok == '-' then
+            if label:sub(1, 1) == '+' then
-                insert(minus_labels, 1, #new_tokens + 1)
+                insert(plus_labels, rl)
+            elseif label:sub(1, 1) == '-' then
+                insert(minus_labels, rl)
             else
                 error('Internal Error: unexpected token for relative label')
             end
@@ -58,29 +96,35 @@ function Preproc:process(tokens)
         if t.tt == 'RELLABEL' then
             t.tt = 'LABEL'
             -- exploits the fact that user labels can't begin with a number
-            t.tok = tostring(i)
+            local name = t.tok:sub(2)
+            t.tok = tostring(i)..name
         elseif t.tt == 'RELLABELSYM' then
             t.tt = 'LABELSYM'
 
-            local rel = t.tok
+            local rel = signs(t.tok)
+            if rel == 0 then
+                error('Internal Error: relative label without signs')
+            end
+            local name = t.tok:sub(abs(rel) + 1)
             local seen = 0
+
             -- TODO: don't iterate over *every* label, just the ones nearby
             if rel > 0 then
-                for _, label_i in ipairs(plus_labels) do
+                for _, rl in ipairs(plus_labels) do
-                    if label_i > i then
+                    if rl.name == name and rl.index > i then
                         seen = seen + 1
                         if seen == rel then
-                            t.tok = tostring(label_i)
+                            t.tok = tostring(rl.index)..name
                             break
                         end
                     end
                 end
             else
-                for _, label_i in ipairs(minus_labels) do
+                for _, rl in ipairs(minus_labels) do
-                    if label_i < i then
+                    if rl.name == name and rl.index < i then
                         seen = seen - 1
                         if seen == rel then
-                            t.tok = tostring(label_i)
+                            t.tok = tostring(rl.index)..name
                             break
                         end
                     end

Lua/lib/lips/util.lua

@@ -16,14 +16,16 @@ local function Class(inherit)
     return setmetatable(class, mt_class)
 end
 
-local function readfile(fn)
+local function readfile(fn, binary)
-    local f = open(fn, 'r')
+    local mode = binary and 'rb' or 'r'
+    local f = open(fn, mode)
     if not f then
-        error('could not open assembly file for reading: '..tostring(fn), 2)
+        local kind = binary and 'binary' or 'assembly'
+        error('could not open '..kind..' file for reading: '..tostring(fn), 2)
     end
-    local asm = f:read('*a')
+    local data = f:read('*a')
     f:close()
-    return asm
+    return data
 end
 
 local function bitrange(x, lower, upper)