lowering of structcopy

7 files changed, 178 insertions, 93 deletions

diff --git a/Makefile b/Makefile
index ef6dcab..0696365 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-SRC=main.c io.c mem.c parse.c lex.c type.c targ.c eval.c ir.c irdump.c abi0.c regalloc.c amd64/sysv.c amd64/isel.c amd64/emit.c obj.c
+SRC=main.c io.c mem.c parse.c lex.c type.c targ.c eval.c ir.c irdump.c intrin.c abi0.c regalloc.c amd64/sysv.c amd64/isel.c amd64/emit.c obj.c
 CFLAGS=-Wall -std=c11 -pedantic
 OBJ=$(patsubst %.c,obj/%.o,$(SRC))
 DEP=$(OBJ:.o=.d)
diff --git a/abi0.c b/abi0.c
index 4c1c702..e523a7d 100644
--- a/abi0.c
+++ b/abi0.c
@@ -178,18 +178,108 @@ patcharg(struct block *blk, int *icall, struct call *call,
       goto Single;
 }
 
+static struct abiarg abiargsbuf[32];
+
+void
+abi0_call(struct function *fn, struct instr *ins, struct block *blk, int *curi)
+{
+   union ref retmem;
+   struct abiargsvec abiargs = {VINIT(abiargsbuf, arraylength(abiargsbuf))};
+   bool sretarghidden = 0;
+   int ni, nf, ns, vararg, nret = 0;
+   struct call *call = &calltab.p[ins->r.i];
+
+   vararg = call->vararg;
+   vinit(&abiargs, abiargsbuf, arraylength(abiargsbuf));
+   ni = nf = ns = 0;
+   assert(!ins->cls == !call->ret.bits);
+   nret = abiret(call->abiret, &abiargs, &ni, call->ret);
+   if (call->ret.isagg) { /* adjust struct return */
+      union irtype retty = call->ret;
+      struct instr alloca = { .cls = KPTR };
+      struct typedata *td = &typedata[retty.dat];
+
+      sretarghidden = ni == 0;
+      alloca.op = Oalloca8 + (td->align == 16);
+      alloca.l = mkref(RICON, td->align == 16 ? 1 : td->siz / 8);
+      retmem = insertinstr(blk, (*curi)++ - call->narg, alloca);
+      if (!nret) /* hidden pointer argument */
+         insertinstr(blk, (*curi)++ - call->narg,
+                     mkinstr(Omove, KPTR, mkref(RREG, abiargs.p[0].reg), retmem));
+   }
+
+   /* adjust args */
+   for (int i = 0, i2 = ni + sretarghidden; i < call->narg; ++i) {
+      union irtype pty = ref2type(instrtab[blk->ins.p[*curi - call->narg + i]].l);
+      int first = abiargs.n;
+      int ret = abiarg(&abiargs, &ni, &nf, &ns, pty);
+      ret = patcharg(blk, curi, call, i, ret, &abiargs.p[first]);
+      if (call->vararg == i) vararg = i2;
+      i2 += ret;
+   }
+   /* adjust return */
+   if (call->ret.bits && !call->ret.isagg) {
+      /* duplicate to reuse same TMP ref */
+      ins->cls = 0;
+      insertinstr(blk, (*curi)++, *ins);
+      /* now ins will be Ocopy <reg> */
+   }
+   if (call->ret.isagg) {
+      replref(fn, blk, (*curi), mkref(RTMP, ins - instrtab), retmem);
+      if (!nret) { /* hidden pointer argument */
+         if (call->abiret[0].reg >= 0)
+            ++nret;
+      } else { /* aggregate returned in regs */
+         union ref r[2];
+         struct instr ins;
+         assert(in_range(nret, 1, 2));
+         ins = mkinstr(Ocopy, call->abiret[0].ty.cls, mkref(RREG, call->abiret[0].reg));
+         r[0] = insertinstr(blk, ++*curi, ins);
+         if (nret == 2) {
+            ins =  mkinstr(Ocopy, call->abiret[1].ty.cls, mkref(RREG, call->abiret[1].reg));
+            r[1] = insertinstr(blk, ++*curi, ins);
+         }
+         for (int i = 0; i < nret; ++i) {
+            struct instr store = {0};
+            /* XXX this can generate unaligned stores */
+            switch (call->abiret[i].ty.cls) {
+            default: assert(0);
+            case KF4: case KI4: store.op = Ostore4; break;
+            case KI8: case KF8: store.op = Ostore8; break;
+            }
+            if (i == 0) {
+               store.l = retmem;
+            } else {
+               store.l = insertinstr(blk, ++*curi,
+                                   mkinstr(Oadd, KPTR, retmem,
+                                           mkref(RICON, cls2siz[call->abiret[0].ty.cls])));
+            }
+            store.r = r[i];
+            insertinstr(blk, ++*curi, store);
+         }
+      }
+   } else if (call->ret.cls) {
+      *ins = mkinstr(Ocopy, call->abiret[0].ty.cls, mkref(RREG, call->abiret[0].reg));
+   }
+
+   if (call->ret.isagg) call->ret = (union irtype){0};
+   call->vararg = vararg;
+   call->abiargregs = alloc(&fn->arena, abiargs.n * sizeof *call->abiargregs, 0);
+   for (int i = 0; i < abiargs.n; ++i) call->abiargregs[i] = abiargs.p[i].reg;
+   call->narg = abiargs.n;
+   vfree(&abiargs);
+}
+
 void
 abi0(struct function *fn)
 {
    uint nparam = typedata[fn->fnty.dat].nmemb;
    const union type *paramty = typedata[fn->fnty.dat].param;
-   static struct abiarg abiargsbuf[32];
    struct abiargsvec abiargs = {VINIT(abiargsbuf, arraylength(abiargsbuf))};
    int rvovar = -1;
    int ni = 0, nf = 0, ns = 0, istart = 0;
    struct block *blk;
    union ref sret = {0};
-   bool sretarghidden = 0;
 
    if (fn->retty.t == TYVOID) {
       fn->nabiret = 0;
@@ -197,7 +287,6 @@ abi0(struct function *fn)
       fn->nabiret = abiret(fn->abiret, &abiargs, &ni, mkirtype(fn->retty));
       if (!fn->nabiret && isagg(fn->retty)) { /* ret agg by hidden pointer */
          struct instr param = copyparam(abiargs.p[0]);
-         sretarghidden = ni == 0;
          sret = insertinstr(fn->entry, 0, param);
          ++istart;
       }
@@ -242,92 +331,9 @@ abi0(struct function *fn)
    do {
       /* adjust calls */
       for (int iinstr = 0; iinstr < blk->ins.n; ++iinstr) {
-         union ref retmem;
          struct instr *ins = &instrtab[blk->ins.p[iinstr]];
-         struct call *call = &calltab.p[ins->r.i];
-         int vararg, nret = 0;
-
          if (ins->op != Ocall) continue;
-
-         vararg = call->vararg;
-         vinit(&abiargs, abiargsbuf, arraylength(abiargsbuf));
-         ni = nf = ns = 0;
-         assert(!ins->cls == !call->ret.bits);
-         nret = abiret(call->abiret, &abiargs, &ni, call->ret);
-         if (call->ret.isagg) { /* adjust struct return */
-            union irtype retty = call->ret;
-            struct instr alloca = { .cls = KPTR };
-            struct typedata *td = &typedata[retty.dat];
-
-            sretarghidden = ni == 0;
-            alloca.op = Oalloca8 + (td->align == 16);
-            alloca.l = mkref(RICON, td->align == 16 ? 1 : td->siz / 8);
-            retmem = insertinstr(blk, iinstr++ - call->narg, alloca);
-            if (!nret) /* hidden pointer argument */
-               insertinstr(blk, iinstr++ - call->narg,
-                           mkinstr(Omove, KPTR, mkref(RREG, abiargs.p[0].reg), retmem));
-         }
-
-         /* adjust args */
-         for (int i = 0, i2 = ni + sretarghidden; i < call->narg; ++i) {
-            union irtype pty = ref2type(instrtab[blk->ins.p[iinstr - call->narg + i]].l);
-            int first = abiargs.n;
-            int ret = abiarg(&abiargs, &ni, &nf, &ns, pty);
-            ret = patcharg(blk, &iinstr, call, i, ret, &abiargs.p[first]);
-            if (call->vararg == i) vararg = i2;
-            i2 += ret;
-         }
-         /* adjust return */
-         if (!call->ret.isagg) {
-            /* duplicate to reuse same TMP ref */
-            ins->cls = 0;
-            insertinstr(blk, iinstr++, *ins);
-            /* now ins will be Ocopy <reg> */
-         }
-         if (call->ret.isagg) {
-            replref(fn, blk, iinstr, mkref(RTMP, ins - instrtab), retmem);
-            if (!nret) { /* hidden pointer argument */
-               if (call->abiret[0].reg >= 0)
-                  ++nret;
-            } else { /* aggregate returned in regs */
-               union ref r[2];
-               struct instr ins;
-               assert(in_range(nret, 1, 2));
-               ins = mkinstr(Ocopy, call->abiret[0].ty.cls, mkref(RREG, call->abiret[0].reg));
-               r[0] = insertinstr(blk, ++iinstr, ins);
-               if (nret == 2) {
-                  ins =  mkinstr(Ocopy, call->abiret[1].ty.cls, mkref(RREG, call->abiret[1].reg));
-                  r[1] = insertinstr(blk, ++iinstr, ins);
-               }
-               for (int i = 0; i < nret; ++i) {
-                  struct instr store = {0};
-                  /* XXX this can generate unaligned stores */
-                  switch (call->abiret[i].ty.cls) {
-                  default: assert(0);
-                  case KF4: case KI4: store.op = Ostore4; break;
-                  case KI8: case KF8: store.op = Ostore8; break;
-                  }
-                  if (i == 0) {
-                     store.l = retmem;
-                  } else {
-                     store.l = insertinstr(blk, ++iinstr,
-                                         mkinstr(Oadd, KPTR, retmem,
-                                                 mkref(RICON, cls2siz[call->abiret[0].ty.cls])));
-                  }
-                  store.r = r[i];
-                  insertinstr(blk, ++iinstr, store);
-               }
-            }
-         } else if (call->ret.cls) {
-            *ins = mkinstr(Ocopy, call->abiret[0].ty.cls, mkref(RREG, call->abiret[0].reg));
-         }
-
-         if (call->ret.isagg) call->ret = (union irtype){0};
-         call->vararg = vararg;
-         call->abiargregs = alloc(&fn->arena, abiargs.n * sizeof *call->abiargregs, 0);
-         for (int i = 0; i < abiargs.n; ++i) call->abiargregs[i] = abiargs.p[i].reg;
-         call->narg = abiargs.n;
-         vfree(&abiargs);
+         abi0_call(fn, ins, blk, &iinstr);
       }
 
       /* adjust returns */
diff --git a/amd64/isel.c b/amd64/isel.c
index 469c56d..58cc49a 100644
--- a/amd64/isel.c
+++ b/amd64/isel.c
@@ -240,12 +240,12 @@ sel(struct function *fn, struct instr *ins, struct block *blk, int *curi)
       break;
    case Oloads1: case Oloadu1: case Oloads2: case Oloadu2:
    case Oloads4: case Oloadu4: case Oloadi8: case Oloadf4: case Oloadf8:
-      if (ins->l.t != RTMP && ins->l.t != RREG)
+      if (ins->l.t != RTMP && ins->l.t != RREG && ins->l.t != RMORE)
          ins->l = insertinstr(blk, (*curi)++, mkinstr(Ocopy, ins->cls, ins->l));
       fuseaddr(fn, &ins->l);
       break;
    case Ostore1: case Ostore2: case Ostore4: case Ostore8:
-      if (ins->l.t != RTMP && ins->l.t != RREG)
+      if (ins->l.t != RTMP && ins->l.t != RREG && ins->l.t != RMORE)
          ins->l = insertinstr(blk, (*curi)++, mkinstr(Ocopy, ins->cls, ins->l));
       fuseaddr(fn, &ins->l);
       fixarg(fn, &ins->r, ins, blk, curi);
diff --git a/intrin.c b/intrin.c
new file mode 100644
index 0000000..6a3bc7f
--- /dev/null
+++ b/intrin.c
@@ -0,0 +1,77 @@
+#include "common.h"
+#include "ir.h"
+
+struct arg { union ref *arg, *ty; };
+
+static int
+intrin(struct block *blk, int *curi, enum intrin in, struct arg *args, int narg, union irtype ret)
+{
+   struct instr *this = &instrtab[blk->ins.p[*curi]];
+   const struct typedata *td;
+   union irtype ty;
+   uint ncopy, step;
+
+   switch (in) {
+   case 0: assert(0);
+   case INstructcopy:
+      assert(narg == 2 && args[0].ty->bits == args[1].ty->bits);
+      ty = ref2type(*args[0].ty);
+      assert(ty.isagg);
+      td = &typedata[ty.cls];
+      step = td->align <= 8 ? td->align : 8;
+      ncopy = td->siz / step;
+      if (ncopy > 4) {
+         enum irclass cls = siz2intcls[cls2siz[KPTR]];
+         /* memcpy */
+         *args[1].ty = *args[0].ty = mktyperef(cls2type(KPTR));
+         insertinstr(blk, (*curi)++, mkarginstr(cls2type(cls), mkintcon(cls, td->siz)));
+         *this = mkinstr(Ocall, 0, mksymref("memcpy"), this->r);
+         calltab.p[this->r.i].narg = 3;
+         calltab.p[this->r.i].ret = cls2type(0);
+         return 0;
+      } else {
+         delinstr(blk, (*curi)--);
+         for (int off = 0; off < td->siz; off += step) {
+            union ref psrc = *args[1].arg, pdst = *args[0].arg, src;
+            if (off) {
+               pdst = mkaddr((struct addr) {.base = *args[0].arg, .disp = off});
+               psrc = mkaddr((struct addr) {.base = *args[1].arg, .disp = off});
+            }
+            src = insertinstr(blk, ++*curi, mkinstr(Oloads1 + 2*ilog2(step), step < 8 ? KI4 : KI8, psrc));
+            insertinstr(blk, ++*curi, mkinstr(Ostore1 + ilog2(step), 0, pdst, src));
+         }
+         return 1;
+      }
+   }
+   assert(0);
+}
+
+void
+lowerintrin(struct function *fn)
+{
+   struct block *blk = fn->entry;
+   static struct arg argsbuf[64];
+   vec_of(struct arg) args = VINIT(argsbuf, arraylength(argsbuf));
+
+   do {
+      for (int i = 0; i < blk->ins.n; ++i) {
+         struct instr *ins = &instrtab[blk->ins.p[i]];
+         if (ins->op == Oarg)
+            vpush(&args, ((struct arg){ &ins->r, &ins->l }));
+         else if (ins->op == Ocall)
+            vinit(&args, argsbuf, arraylength(argsbuf));
+         else if (ins->op == Ointrin) {
+            int arg0 = i - args.n;
+            assert(calltab.p[ins->r.i].narg == args.n);
+            if (intrin(blk, &i, ins->l.i, args.p, args.n, calltab.p[ins->r.i].ret))
+               for (int j = args.n; j > 0; --j, --i)
+                  delinstr(blk, arg0);
+            else
+               abi0_call(fn, ins, blk, &i);
+            vinit(&args, argsbuf, arraylength(argsbuf));
+         } else if (ins->op != Omove) assert(args.n == 0);
+      }
+   } while ((blk = blk->lnext) != fn->entry);
+}
+
+/* vim:set ts=3 sw=3 expandtab: */
diff --git a/ir.c b/ir.c
index 5b54d70..432a10d 100644
--- a/ir.c
+++ b/ir.c
@@ -218,7 +218,7 @@ newinstr(void)
 {
    if (instrfreelist != -1) {
       int t = instrfreelist;
-      memcpy(&instrfreelist, &instrtab[instrfreelist], sizeof(int));
+      memcpy(&instrfreelist, &instrtab[t], sizeof(int));
       return t;
    }
    assert(ninstr < arraylength(instrtab));
@@ -257,7 +257,7 @@ delinstr(struct block *blk, int idx)
 {
    assert(idx >= 0 && idx < blk->ins.n);
    memcpy(&instrtab[blk->ins.p[idx]], &instrfreelist, sizeof(int));
-   instrfreelist = idx;
+   instrfreelist = blk->ins.p[idx];
    for (int i = idx; i < blk->ins.n; ++i)
       blk->ins.p[i] = blk->ins.p[i + 1];
    --blk->ins.n;
@@ -401,6 +401,7 @@ irfini(struct function *fn)
    extern int nerror;
    if (!nerror) {
       abi0(fn);
+      lowerintrin(fn);
       mctarg->isel(fn);
       regalloc(fn);
       if (!ccopt.dbg.any)
diff --git a/ir.h b/ir.h
index 7ff31c8..1182ed3 100644
--- a/ir.h
+++ b/ir.h
@@ -219,7 +219,9 @@ void replref(struct function *, struct block *, int, union ref from, union ref t
 
 void irdump(struct function *);
 
+void lowerintrin(struct function *);
 void abi0(struct function *);
+void abi0_call(struct function *, struct instr *, struct block *blk, int *curi);
 void regalloc(struct function *);
 
 /* vim:set ts=3 sw=3 expandtab: */
diff --git a/todo.txt b/todo.txt
index fdb7cf0..f7d4de9 100644
--- a/todo.txt
+++ b/todo.txt
@@ -5,7 +5,6 @@ Things to finish before moving onto compiler optimizations, C extensions, other
 - frontend: try to repr vars as ssa temps until they have their address taken or are mutated (to reduce no. of mem instrs)??
 - ir: implement mem2reg
 - backend: implement ELF object output (ELF header, relocations, etc)
-- backend: figure out how and when to lower intrinsincs (currently just structcopy..)
 - frontend: finish C impl: initializers, preprocessor (#include, fn-like macros, etc), forward declarations, for, switch, break-continue, goto
 
 at some point add another backend like arm64 to make sure the non target specific stuff is generic enough..