reimplement Dumper; lips is mostly working again

lips/Collector.lua

@@ -85,7 +85,7 @@ function Collector:directive()
         insert(self.statements, self:statement('!'..kind, ...))
     end
     if name == 'ORG' then
-        add(name, self:const(false, true))
+        add(name, self:const(nil, 'no labels'))
     elseif name == 'ALIGN' or name == 'SKIP' then
         if self:is_EOL() and name == 'ALIGN' then
             add(name, self:token('NUM', 0))

lips/Dumper.lua

@@ -108,7 +108,7 @@ function Dumper:desym(t)
     elseif t.tt == 'LABELSYM' or t.tt == 'LABELREL' then
         local label = self.labels[t.tok]
         if label == nil then
-            self:error('undefined label')
+            self:error('undefined label', t.tok)
         end
         if t.offset then
             label = label + t.offset
@@ -144,12 +144,6 @@ function Dumper:toval(t)
     if t.negate then
         val = -val
     end
-    if t.negate or t.signed then
-        if val >= 0x10000 or val < -0x8000 then
-            self:error('value out of range')
-        end
-        val = val % 0x10000
-    end
 
     if t.portion == 'upper' then
         val = bitrange(val, 16, 31)
@@ -165,6 +159,13 @@ function Dumper:toval(t)
         val = upper
     end
 
+    if t.negate or t.signed then
+        if val >= 0x10000 or val < -0x8000 then
+            self:error('value out of range', val)
+        end
+        val = val % 0x10000
+    end
+
     return val
 end
 
@@ -250,7 +251,6 @@ function Dumper:assemble_r(first, out)
     return s
 end
 
--- NOTE: we could move format_{in,out} to its own virtual class and inherit it
 function Dumper:format_in(informat)
     -- see data.lua for a guide on what all these mean
     local args = {}
@@ -277,17 +277,17 @@ function Dumper:format_in(informat)
         elseif c == 'Z' and not args.rt then
             args.rt = self:register(data.sys_registers)
         elseif c == 'o' and not args.offset then
-            args.offset = Token(self:const()):set('signed')
+            args.offset = self:const():set('signed')
         elseif c == 'r' and not args.offset then
-            args.offset = Token(self:const('relative')):set('signed')
+            args.offset = self:const('relative'):set('signed')
         elseif c == 'i' and not args.immediate then
             args.immediate = self:const(nil, 'no label')
         elseif c == 'I' and not args.index then
-            args.index = Token(self:const()):set('index')
+            args.index = self:const():set('index')
         elseif c == 'k' and not args.immediate then
-            args.immediate = Token(self:const(nil, 'no label')):set('negate')
+            args.immediate = self:const(nil, 'no label'):set('negate')
         elseif c == 'K' and not args.immediate then
-            args.immediate = Token(self:const(nil, 'no label')):set('signed')
+            args.immediate = self:const(nil, 'no label'):set('signed')
         elseif c == 'b' and not args.base then
             args.base = self:deref():set('tt', 'REG')
         else
@@ -356,9 +356,21 @@ local assembled_directives = {
     ['!ORG'] = true,
 }
 
+function Dumper:fill(length, content)
+    self:validate(content, 8)
+    local bytes = {}
+    for i=1, length do
+        insert(bytes, content)
+    end
+    local t = Token(self.fn, self.line, 'BYTES', bytes)
+    local s = Statement(self.fn, self.line, '!DATA', t)
+    return s
+end
+
 function Dumper:load(statements)
     self.labels = {}
 
+    local pos = 0
     local new_statements = {}
     for i=1, #statements do
         local s = statements[i]
@@ -366,16 +378,57 @@ function Dumper:load(statements)
         self.line = s.line
         if s.type:sub(1, 1) == '!' then
             if s.type == '!LABEL' then
-                self.labels[s[1].tok] = i
+                self.labels[s[1].tok] = pos
             elseif s.type == '!DATA' then
-                -- noop
+                pos = pos + util.measure_data(s)
+                insert(new_statements, s)
+            elseif s.type == '!ORG' then
+                pos = s[1].tok
+                insert(new_statements, s)
+            elseif s.type == '!ALIGN' or s.type == '!SKIP' then
+                local length, content
+                if s.type == '!ALIGN' then
+                    local align = s[1] and s[1].tok or 2
+                    content = s[2] and s[2].tok or 0
+                    if align < 0 then
+                        self:error('negative alignment')
+                    else
+                        align = 2^align
+                    end
+                    local temp = pos + align - 1
+                    length = temp - (temp % align) - pos
+                else
+                    length = s[1] and s[1].tok or 0
+                    content = s[2] and s[2].tok or nil
+                end
+
+                if content == nil then
+                    pos = pos + length
+                    local new = Statement(self.fn, self.line, '!ORG', pos)
+                    insert(new_statements, new)
+                elseif length > 0 then
+                    insert(new_statements, self:fill(length, content))
+                elseif length < 0 then
+                    pos = pos + length
+                    local new = Statement(self.fn, self.line, '!ORG', pos)
+                    insert(new_statements, new)
+                    insert(new_statements, self:fill(length, content))
+                    local new = Statement(self.fn, self.line, '!ORG', pos)
+                    insert(new_statements, new)
+                else
+                    -- length is 0, noop
+                end
             else
-                -- TODO: internal error?
+                error('Internal Error: unknown statement, got '..s.type)
-                self:error('unknown statement', s.type)
             end
+        else
+            pos = pos + 4
+            insert(new_statements, s)
         end
     end
 
+    statements = new_statements
+    new_statements = {}
     -- TODO: keep track of lengths here?
     self.pos = 0
     for i=1, #statements do
@@ -383,19 +436,29 @@ function Dumper:load(statements)
         self.fn = s.fn
         self.line = s.line
         if s.type:sub(1, 1) ~= '!' then
-            s = self:assemble(s)
+            local new = self:assemble(s)
+            insert(new_statements, new)
+            self.pos = self.pos + 4
+        elseif s.type == '!DATA' then
+            for i, t in ipairs(s) do
+                if t.tt == 'LABEL' then
+                    local label = self.labels[t.tok]
+                    if label == nil then
+                        self:error('undefined label', t.tok)
+                    end
+                    t.tt = 'WORDS'
+                    t.tok = {self.labels[t.tok]}
+                end
+            end
             insert(new_statements, s)
-            self.pos = self.pos + 4 -- FIXME: assumes no pseudo-ops
+            self.pos = self.pos + util.measure_data(s)
-        elseif assembled_directives[s.type] then
+        elseif s.type == '!ORG' then
-            -- FIXME: check for LABELs in !DATA
-            -- TODO: reimplement ALIGN and SKIP here
             insert(new_statements, s)
-            self.pos = self.pos + s:length()
+            self.pos = s[1].tok
         elseif s.type == '!LABEL' then
             -- noop
         else
-            print(s.type)
+            error('Internal Error: unknown statement, got '..s.type)
-            error('Internal Error: unknown statement found in Dumper')
         end
     end
 
@@ -426,7 +489,7 @@ function Dumper:dump()
                     end
                 elseif t.tt == 'BYTES' then
                     for _, b in ipairs(t.tok) do
-                        local b0 = bitrange(h, 0, 7)
+                        local b0 = bitrange(b, 0, 7)
                         self:write{b0}
                     end
                 else
@@ -434,7 +497,7 @@ function Dumper:dump()
                 end
             end
         elseif s.type == '!ORG' then
-            self.pos = s[1]
+            self.pos = s[1].tok
         end
     end
 

lips/Parser.lua

@@ -98,29 +98,40 @@ function Parser:tokenize(asm)
     self.statements = collector:collect(tokens, self.main_fn)
 end
 
+function Parser:debug_dump()
+    for i, s in ipairs(self.statements) do
+        local values = ''
+        for j, v in ipairs(s) do
+            local tok = v.tok
+            if type(tok) == 'number' then
+                tok = ("$%X"):format(tok)
+            end
+            values = values..'\t'..v.tt..'('..tostring(tok)..')'
+        end
+        values = values:sub(2)
+        print(s.line, s.type, values)
+    end
+end
+
 function Parser:parse(asm)
     self:tokenize(asm)
 
+    if self.options.debug_token then self:debug_dump() end
+
     local preproc = Preproc(self.options)
     self.statements = preproc:process(self.statements)
     self.statements = preproc:expand(self.statements)
 
-    -- DEBUG
+    if self.options.debug_pre then self:debug_dump() end
-    for i, s in ipairs(self.statements) do
-        local values = ''
-        for j, v in ipairs(s) do
-            values = values..'\t'..v.tt..'('..tostring(v.tok)..')'
-        end
-        values = values:sub(2)
-        print(i, s.type, values)
-    end
 
     local dumper = Dumper(self.writer, self.options)
     self.statements = dumper:load(self.statements)
 
-    --if self.options.labels then
+    if self.options.debug_dump then self:debug_dump() end
-    --    dumper:export_labels(self.options.labels)
+
-    --end
+    if self.options.labels then
+        dumper:export_labels(self.options.labels)
+    end
     return dumper:dump()
 end
 

lips/Reader.lua

@@ -56,18 +56,18 @@ function Reader:register(registers)
 end
 
 function Reader:const(relative, no_label)
-    if no_label then
+    self:expect{'NUM', 'LABELSYM', 'LABELREL'}
-        self:expect{'NUM'}
-    else
-        self:expect{'NUM', 'LABELSYM'}
-    end
     local t = self.s[self.i]
+    -- overrides will want to LUI a label; let portioned labels pass
+    if no_label and not t.portion then
+        self:expect{'NUM', 'LABELREL'}
+    end
     local new = Token(t)
     if relative then
         if t.tt == 'LABELSYM' then
-            new.t = 'LABELREL'
+            new.tt = 'LABELREL'
-        else
+        elseif t.tt == 'NUM' then
-            new.t = 'REL'
+            new.tt = 'REL'
         end
     end
     return new

lips/Statement.lua

@@ -51,9 +51,4 @@ function Statement:validate(n)
     end
 end
 
--- TODO: just move push_data into Statement too
-function Statement:length()
-    assert(self.type == '!DATA', 'Statement:length only works on !DATA types')
-end
-
 return Statement

lips/overrides.lua

@@ -88,7 +88,7 @@ function overrides.LA(self, name)
     local immediate = self:token(im):set('portion', 'upperoff')
     self:push_new('LUI', rt, immediate)
     local immediate = self:token(im):set('portion', 'lower')
-    self:push_new('ADDIU', rt, immediate)
+    self:push_new('ADDIU', rt, rt, immediate)
 end
 
 function overrides.PUSH(self, name)

lips/util.lua

@@ -39,9 +39,27 @@ else -- 5.2, 5.3
     end
 end
 
+local function measure_data(s)
+    assert(s and s.type == '!DATA', 'Internal Error: expected !DATA statement')
+    local n = 0
+    for i, t in ipairs(s) do
+        if t.type == 'LABEL' then
+            n = n + 4
+        elseif t.type == 'WORDS' then
+            n = n + #t.tok * 4
+        elseif t.type == 'HALFWORDS' then
+            n = n + #t.tok * 2
+        elseif t.type == 'BYTES' then
+            n = n + #t.tok * 1
+        end
+    end
+    return n
+end
+
 return {
     readfile = readfile,
     bitrange = bitrange,
     parent = parent,
     loadcode = loadcode,
+    measure_data = measure_data,
 }