delete tool/instruction.rb (2nd try)

Previous commit changed insns.def format. Now is the time for its
generators. In doing so I chose to modernize the system, not just
patch.  My attempt includes

- extensive use of Onigumo regular expressions
- split from one big file (instruction.rb) into separated MVC
- partial view

Also, let me take this opportunity to kill old unused features
such as

- stack caching
- minsns / yasmdata which are never seriously used
- yarvarch document generation (moved to doc/)
- vast majority of unused arguments to insns2vm.rb

This commit generates VM source codes that cleanly compile, and
the generated binary passes tests.  At least for me.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@61784 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

tool/ruby_vm/models/attribute.rb

@@ -0,0 +1,44 @@
+#! /your/favourite/path/to/ruby
+# -*- mode: ruby; coding: utf-8; indent-tabs-mode: nil; ruby-indent-level: 2 -*-
+# -*- frozen_string_literal: true; -*-
+# -*- warn_indent: true; -*-
+#
+# Copyright (c) 2017 Urabe, Shyouhei.  All rights reserved.
+#
+# This file is  a part of the programming language  Ruby.  Permission is hereby
+# granted, to  either redistribute and/or  modify this file, provided  that the
+# conditions  mentioned in  the file  COPYING are  met.  Consult  the file  for
+# details.
+
+require_relative 'c_expr'
+
+class RubyVM::Attribute
+  include RubyVM::CEscape
+  attr_reader :insn, :key, :type, :expr
+
+  def initialize insn:, name:, type:, location:, expr:
+    @insn = insn
+    @key = name
+    @expr = RubyVM::CExpr.new location: location, expr: expr
+    @type = type
+  end
+
+  def name
+    as_tr_cpp "attr #{@key} @ #{@insn.name}"
+  end
+
+  def pretty_name
+    "attr #{type} #{key} @ #{insn.pretty_name}"
+  end
+
+  def declaration
+    argv = @insn.opes.map {|o| o[:decl] }.join(', ')
+    sprintf '%s %s(%s)', @type, name, argv
+  end
+
+  def definition
+    argv = @insn.opes.map {|o| "MAYBE_UNUSED(#{o[:decl]})" }.join(",\n    ")
+    argv = "\n    #{argv}\n" if @insn.opes.size > 1
+    sprintf "%s\n%s(%s)", @type, name, argv
+  end
+end

tool/ruby_vm/models/bare_instructions.rb

@@ -0,0 +1,162 @@
+#! /your/favourite/path/to/ruby
+# -*- mode: ruby; coding: utf-8; indent-tabs-mode: nil; ruby-indent-level: 2 -*-
+# -*- frozen_string_literal: true; -*-
+# -*- warn_indent: true; -*-
+#
+# Copyright (c) 2017 Urabe, Shyouhei.  All rights reserved.
+#
+# This file is  a part of the programming language  Ruby.  Permission is hereby
+# granted, to  either redistribute and/or  modify this file, provided  that the
+# conditions  mentioned in  the file  COPYING are  met.  Consult  the file  for
+# details.
+
+require_relative '../loaders/insns_def'
+require_relative 'c_expr'
+require_relative 'typemap'
+require_relative 'attribute'
+
+class RubyVM::BareInstructions
+  attr_reader :template, :name, :opes, :pops, :rets, :decls, :expr
+
+  def initialize template:, name:, location:, signature:, attributes:, expr:
+    @template = template
+    @name     = name
+    @loc      = location
+    @sig      = signature
+    @expr     = RubyVM::CExpr.new expr
+    @opes     = typesplit @sig[:ope]
+    @pops     = typesplit @sig[:pop].reject {|i| i == '...' }
+    @rets     = typesplit @sig[:ret].reject {|i| i == '...' }
+    @attrs    = attributes.map {|i|
+      RubyVM::Attribute.new insn: self, **i
+    }.each_with_object({}) {|a, h|
+      h[a.key] = a
+    }
+    @attrs_orig = @attrs.dup
+  end
+
+  def pretty_name
+    n = @sig[:name]
+    o = @sig[:ope].map{|i| i[/\S+$/] }.join ', '
+    p = @sig[:pop].map{|i| i[/\S+$/] }.join ', '
+    r = @sig[:ret].map{|i| i[/\S+$/] }.join ', '
+    return sprintf "%s(%s)(%s)(%s)", n, o, p, r
+  end
+
+  def bin
+    return "BIN(#{name})"
+  end
+
+  def call_attribute name
+    return sprintf 'CALL_ATTRIBUTE(%s)', [
+      name, @name, @opes.map {|i| i[:name] }
+    ].flatten.compact.join(', ')
+  end
+
+  def sp_inc
+    return @attrs.fetch "sp_inc" do |k|
+      return generate_attribute k, 'rb_snum_t', rets.size - pops.size
+    end
+  end
+
+  def has_attribute? k
+    @attrs_orig.has_key? k
+  end
+
+  def attributes
+    # need to generate predefined attribute defaults
+    sp_inc
+    # other_attribute
+    # ...
+    return @attrs.values
+  end
+
+  def width
+    return 1 + opes.size
+  end
+
+  def declarations
+    return @variables                                        \
+      . values                                               \
+      . group_by {|h| h[:type] }                             \
+      . map      {|t, v| [t, v.map {|i| i[:name] }.sort ] }  \
+      . map      {|t, v| sprintf("%s %s", t, v.join(', ')) } \
+      . sort
+  end
+
+  def preamble
+    # preamble makes sense for operand unifications
+    return []
+  end
+
+  def sc?
+    # sc stands for stack caching.
+    return false
+  end
+
+  def cast_to_VALUE var, expr = var[:name]
+    RubyVM::Typemap.typecast_to_VALUE var[:type], expr
+  end
+
+  def cast_from_VALUE var, expr = var[:name]
+    RubyVM::Typemap.typecast_from_VALUE var[:type], expr
+  end
+
+  def operands_info
+    opes.map {|o|
+      c, _ = RubyVM::Typemap.fetch o[:type]
+      next c
+    }.join
+  end
+
+  def pushs_frame?
+    opes.any? {|o| /CALL_INFO/ =~ o[:type] }
+  end
+
+  def inspect
+    sprintf "#<%s@%s:%d>", @name, @loc[0], @loc[1]
+  end
+
+  private
+
+  def generate_attribute k, t, v
+    attr = RubyVM::Attribute.new \
+      insn: self, \
+      name: k, \
+      type: t, \
+      location: [], \
+      expr: v.to_s + ';'
+    return @attrs[k] = attr
+  end
+
+  def typesplit a
+    @variables ||= {}
+    a.map do |decl|
+      md = %r'
+        (?<comment> /[*] [^*]* [*]+ (?: [^*/] [^*]* [*]+ )* / ){0}
+        (?<ws>      \g<comment> | \s                          ){0}
+        (?<ident>   [_a-zA-Z] [0-9_a-zA-Z]*                   ){0}
+        (?<type>    (?: \g<ident> \g<ws>+ )* \g<ident>        ){0}
+        (?<var>     \g<ident>                                 ){0}
+        \G          \g<ws>* \g<type> \g<ws>+ \g<var>
+      'x.match(decl)
+      @variables[md['var']] ||= {
+        decl: decl,
+        type: md['type'],
+        name: md['var'],
+      }
+    end
+  end
+
+  @instances = RubyVM::InsnsDef.map {|h| new template: h, **h }
+
+  def self.fetch name
+    @instances.find do |insn|
+      insn.name == name
+    end or raise IndexError, "instruction not found: #{name}"
+  end
+
+  def self.to_a
+    @instances
+  end
+end

tool/ruby_vm/models/c_expr.rb

@@ -0,0 +1,41 @@
+#! /your/favourite/path/to/ruby
+# -*- mode: ruby; coding: utf-8; indent-tabs-mode: nil; ruby-indent-level: 2 -*-
+# -*- frozen_string_literal: true; -*-
+# -*- warn_indent: true; -*-
+#
+# Copyright (c) 2017 Urabe, Shyouhei.  All rights reserved.
+#
+# This file is  a part of the programming language  Ruby.  Permission is hereby
+# granted, to  either redistribute and/or  modify this file, provided  that the
+# conditions  mentioned in  the file  COPYING are  met.  Consult  the file  for
+# details.
+
+require_relative '../helpers/c_escape.rb'
+
+class RubyVM::CExpr
+  include RubyVM::CEscape
+
+  attr_reader :__FILE__, :__LINE__, :expr
+
+  def initialize location:, expr:
+    @__FILE__  = location[0]
+    @__LINE__  = location[1]
+    @expr      = expr
+  end
+
+  # blank, in sense of C program.
+  RE = %r'\A{\g<s>*}\z|\A(?<s>\s|/[*][^*]*[*]+([^*/][^*]*[*]+)*/)*\z'
+  if RUBY_VERSION > '2.4' then
+    def blank?
+      RE.match? @expr
+    end
+  else
+    def blank?
+      RE =~ @expr
+    end
+  end
+
+  def inspect
+    sprintf "#<%s:%d %s>", @__FILE__, @__LINE__, @expr
+  end
+end

tool/ruby_vm/models/instructions.rb

@@ -0,0 +1,22 @@
+#! /your/favourite/path/to/ruby
+# -*- mode: ruby; coding: utf-8; indent-tabs-mode: nil; ruby-indent-level: 2 -*-
+# -*- frozen_string_literal: true; -*-
+# -*- warn_indent: true; -*-
+#
+# Copyright (c) 2017 Urabe, Shyouhei.  All rights reserved.
+#
+# This file is  a part of the programming language  Ruby.  Permission is hereby
+# granted, to  either redistribute and/or  modify this file, provided  that the
+# conditions  mentioned in  the file  COPYING are  met.  Consult  the file  for
+# details.
+
+require_relative 'bare_instructions'
+require_relative 'operands_unifications'
+require_relative 'instructions_unifications'
+
+RubyVM::Instructions = RubyVM::BareInstructions.to_a + \
+                       RubyVM::OperandsUnifications.to_a + \
+                       RubyVM::InstructionsUnifications.to_a
+
+require_relative 'trace_instructions'
+RubyVM::Instructions.freeze

tool/ruby_vm/models/instructions_unifications.rb

@@ -0,0 +1,43 @@
+#! /your/favourite/path/to/ruby
+# -*- mode: ruby; coding: utf-8; indent-tabs-mode: nil; ruby-indent-level: 2 -*-
+# -*- frozen_string_literal: true; -*-
+# -*- warn_indent: true; -*-
+#
+# Copyright (c) 2017 Urabe, Shyouhei.  All rights reserved.
+#
+# This file is  a part of the programming language  Ruby.  Permission is hereby
+# granted, to  either redistribute and/or  modify this file, provided  that the
+# conditions  mentioned in  the file  COPYING are  met.  Consult  the file  for
+# details.
+
+require_relative '../helpers/c_escape'
+require_relative '../loaders/opt_insn_unif_def'
+require_relative 'bare_instructions'
+
+class RubyVM::InstructionsUnifications
+  include RubyVM::CEscape
+
+  attr_reader :name
+
+  def initialize location:, signature:
+    @location = location
+    @name     = namegen signature
+    @series   = signature.map do |i|
+      RubyVM::BareInstructions.fetch i # Misshit is fatal
+    end
+  end
+
+  private
+
+  def namegen signature
+    as_tr_cpp ['UNIFIED', *signature].join('_')
+  end
+
+  @instances = RubyVM::OptInsnUnifDef.map do |h|
+    new(**h)
+  end
+
+  def self.to_a
+    @instances
+  end
+end

tool/ruby_vm/models/operands_unifications.rb

@@ -0,0 +1,137 @@
+#! /your/favourite/path/to/ruby
+# -*- mode: ruby; coding: utf-8; indent-tabs-mode: nil; ruby-indent-level: 2 -*-
+# -*- frozen_string_literal: true; -*-
+# -*- warn_indent: true; -*-
+#
+# Copyright (c) 2017 Urabe, Shyouhei.  All rights reserved.
+#
+# This file is  a part of the programming language  Ruby.  Permission is hereby
+# granted, to  either redistribute and/or  modify this file, provided  that the
+# conditions  mentioned in  the file  COPYING are  met.  Consult  the file  for
+# details.
+
+require_relative '../helpers/c_escape'
+require_relative '../loaders/opt_operand_def'
+require_relative 'bare_instructions'
+
+class RubyVM::OperandsUnifications < RubyVM::BareInstructions
+  include RubyVM::CEscape
+
+  attr_reader :preamble, :original, :spec
+
+  def initialize location:, signature:
+    name             = signature[0]
+    @original        = RubyVM::BareInstructions.fetch name
+    template         = @original.template
+    parts            = compose location, signature, template[:signature]
+    json             = template.dup
+    json[:location]  = location
+    json[:signature] = parts[:signature]
+    json[:name]      = parts[:name]
+    @preamble        = parts[:preamble]
+    @spec            = parts[:spec]
+    super template: template, **json
+    parts[:vars].each do |v|
+      @variables[v[:name]] ||= v
+    end
+  end
+
+  def operand_shift_of var
+    before = @original.opes.find_index var
+    after  = @opes.find_index var
+    raise "no #{var} for #{@name}" unless before and after
+    return before - after
+  end
+
+  def condition ptr
+    # :FIXME: I'm not sure if this method should be in model?
+    exprs = @spec.each_with_index.map do |(var, val), i|
+      case val when '*' then
+        next nil
+      else
+        type = @original.opes[i][:type]
+        expr = RubyVM::Typemap.typecast_to_VALUE type, val
+        next "#{ptr}[#{i}] == #{expr}"
+      end
+    end
+    exprs.compact!
+    if exprs.size == 1 then
+      return exprs[0]
+    else
+      exprs.map! {|i| "(#{i})" }
+      return exprs.join ' && '
+    end
+  end
+
+  private
+
+  def namegen signature
+    insn, argv = *signature
+    wcary = argv.map do |i|
+      case i when '*' then
+        'WC'
+      else
+        i
+      end
+    end
+    as_tr_cpp [insn, *wcary].join(', ')
+  end
+
+  def compose location, spec, template
+    name    = namegen spec
+    *, argv = *spec
+    opes    = @original.opes
+    if opes.size != argv.size
+      raise sprintf("operand size mismatch for %s (%s's: %d, given: %d)",
+                    name, template[:name], opes.size, argv.size)
+    else
+      src  = []
+      mod  = []
+      spec = []
+      vars = []
+      argv.each_index do |i|
+        j = argv[i]
+        k = opes[i]
+        spec[i] = [k, j]
+        case j when '*' then
+          # operand is from iseq
+          mod << k[:decl]
+        else
+          # operand is inside C
+          vars << k
+          src << {
+            location: location,
+            expr: "    #{k[:name]} = #{j};"
+          }
+        end
+      end
+      src.map! {|i| RubyVM::CExpr.new i }
+      return {
+        name: name,
+        signature: {
+          name: name,
+          ope: mod,
+          pop: template[:pop],
+          ret: template[:ret],
+        },
+        preamble: src,
+        vars: vars,
+        spec: spec
+      }
+    end
+  end
+
+  @instances = RubyVM::OptOperandDef.map do |h|
+    new(**h)
+  end
+
+  def self.to_a
+    @instances
+  end
+
+  def self.each_group
+    to_a.group_by(&:original).each_pair do |k, v|
+      yield k, v
+    end
+  end
+end

tool/ruby_vm/models/trace_instructions.rb

@@ -0,0 +1,71 @@
+#! /your/favourite/path/to/ruby
+# -*- mode: ruby; coding: utf-8; indent-tabs-mode: nil; ruby-indent-level: 2 -*-
+# -*- frozen_string_literal: true; -*-
+# -*- warn_indent: true; -*-
+#
+# Copyright (c) 2017 Urabe, Shyouhei.  All rights reserved.
+#
+# This file is  a part of the programming language  Ruby.  Permission is hereby
+# granted, to  either redistribute and/or  modify this file, provided  that the
+# conditions  mentioned in  the file  COPYING are  met.  Consult  the file  for
+# details.
+
+require_relative '../helpers/c_escape'
+require_relative 'bare_instructions'
+
+class RubyVM::TraceInstructions
+  include RubyVM::CEscape
+
+  attr_reader :name
+
+  def initialize orig:
+    @orig = orig
+    @name = as_tr_cpp "trace @ #{@orig.name}"
+  end
+
+  def pretty_name
+    return sprintf "%s(...)(...)(...)", @name
+  end
+
+  def jump_destination
+    return @orig.name
+  end
+
+  def bin
+    return sprintf "BIN(%s)", @name
+  end
+
+  def width
+    return @orig.width
+  end
+
+  def operands_info
+    return @orig.operands_info
+  end
+
+  def rets
+    return ['...']
+  end
+
+  def pops
+    return ['...']
+  end
+
+  def attributes
+    return []
+  end
+
+  def has_attribute? *;
+    return false
+  end
+
+  private
+
+  @instances = RubyVM::Instructions.map {|i| new orig: i }
+
+  def self.to_a
+    @instances
+  end
+
+  RubyVM::Instructions.push(*to_a)
+end

tool/ruby_vm/models/typemap.rb

@@ -0,0 +1,61 @@
+#! /your/favourite/path/to/ruby
+# -*- mode: ruby; coding: utf-8; indent-tabs-mode: nil; ruby-indent-level: 2 -*-
+# -*- frozen_string_literal: true; -*-
+# -*- warn_indent: true; -*-
+#
+# Copyright (c) 2017 Urabe, Shyouhei.  All rights reserved.
+#
+# This file is  a part of the programming language  Ruby.  Permission is hereby
+# granted, to  either redistribute and/or  modify this file, provided  that the
+# conditions  mentioned in  the file  COPYING are  met.  Consult  the file  for
+# details.
+
+RubyVM::Typemap = {
+  "..."            => %w[. TS_VARIABLE],
+  "CALL_CACHE"     => %w[E TS_CALLCACHE],
+  "CALL_INFO"      => %w[C TS_CALLINFO],
+  "CDHASH"         => %w[H TS_CDHASH],
+  "GENTRY"         => %w[G TS_GENTRY],
+  "IC"             => %w[K TS_IC],
+  "ID"             => %w[I TS_ID],
+  "ISEQ"           => %w[S TS_ISEQ],
+  "OFFSET"         => %w[O TS_OFFSET],
+  "VALUE"          => %w[V TS_VALUE],
+  "lindex_t"       => %w[L TS_LINDEX],
+  "rb_insn_func_t" => %w[F TS_FUNCPTR],
+  "rb_num_t"       => %w[N TS_NUM],
+}
+
+# :FIXME: should this method be here?
+class << RubyVM::Typemap
+  def typecast_from_VALUE type, val
+    # see also iseq_set_sequence()
+    case type
+    when '...'
+      raise "cast not possible: #{val}"
+    when 'VALUE' then
+      return val
+    when 'rb_num_t', 'lindex_t' then
+      return "NUM2LONG(#{val})"
+    when 'ID' then
+      return "SYM2ID(#{val})"
+    else
+      return "(#{type})(#{val})"
+    end
+  end
+
+  def typecast_to_VALUE type, val
+    case type
+    when 'VALUE' then
+      return val
+    when 'ISEQ', 'rb_insn_func_t' then
+      return "(VALUE)(#{val})"
+    when 'rb_num_t', 'lindex_t'
+      "LONG2NUM(#{val})"
+    when 'ID' then
+      return "ID2SYM(#{val})"
+    else
+      raise ":FIXME: TBW for #{type}"
+    end
+  end
+end