Add benchmark workflow

.github/workflows/benchmark.yml

@@ -0,0 +1,70 @@
+name: Benchmark
+
+on:
+  push:
+    branches:
+      - master
+
+permissions:
+  contents: read
+
+# Run sequentially: each benchmark fetches the parent commit's data.json
+# from the previous deploy, so we must not race a still-deploying run.
+concurrency:
+  group: benchmark-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+          bundler-cache: true
+
+      - id: pages
+        uses: actions/configure-pages@v6
+
+      # Try parent commit's snapshot first (history continuity), then the
+      # latest data.json (fallback for forks / first push), otherwise keep
+      # the committed template ({"entries": []}). curl -fsSL leaves the
+      # existing file untouched on HTTP errors.
+      - name: Fetch parent's data (with fallback to latest data.json)
+        run: |
+          curl -fsSL "${{ steps.pages.outputs.base_url }}/${{ github.event.before }}.json" \
+            -o tool/site/data.json || \
+          curl -fsSL "${{ steps.pages.outputs.base_url }}/data.json" \
+            -o tool/site/data.json || true
+
+      - name: Run benchmarks
+        run: |
+          bundle exec ruby tool/benchmarks/typeprof.rb
+          bundle exec ruby tool/benchmarks/optcarrot.rb
+          bundle exec ruby tool/benchmarks/redmine.rb
+
+      - name: Update site data
+        run: |
+          ruby tool/update_site_data.rb tool/site/data.json \
+            tmp/typeprof_result.json \
+            tmp/optcarrot_result.json \
+            tmp/redmine_result.json
+
+      - uses: actions/upload-pages-artifact@v5
+        with:
+          path: tool/site
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-slim
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@v5

.gitignore

@@ -1,4 +1,5 @@
-/tmp/
+/tmp/*
+!/tmp/.keep
 /pkg/
 dog_bench.stackprof.dump
 dog_bench.pf2profile

tool/benchmark_helper.rb

@@ -0,0 +1,80 @@
+require "json"
+require "bundler"
+
+module TypeProf
+  module Benchmark
+    TMP_DIR = File.expand_path("../tmp", __dir__)
+    TYPEPROF_BIN = File.expand_path("../bin/typeprof", __dir__)
+
+    # coverage_per_slot:   percentage of type slots (each method's argument/return) inferred
+    # coverage_per_method: percentage of methods whose all slots are inferred (fully typed)
+    Result = Data.define(:name, :elapsed, :coverage_per_slot, :coverage_per_method)
+
+    class << self
+      def run(name:, repo:, ref:, targets: ["."], exclude: [], setup: nil)
+        workspace = clone_if_needed(name, repo, ref)
+        Dir.chdir(workspace) do
+          # Strip typeprof's bundle env (BUNDLE_GEMFILE, RUBYOPT=-rbundler/setup
+          # injected by `bundle exec`) so subprocesses below — setup commands
+          # like `bundle add` / `rbs collection install`, and the typeprof
+          # subprocess — see only the target project's own bundle context.
+          # Without this, typeprof's rbs version leaks into rbs collection
+          # install and produces a lock incompatible with the runtime rbs.
+          Bundler.with_unbundled_env do
+            setup&.call
+            result = run_typeprof(name: name, targets: targets, exclude: exclude)
+            json = JSON.pretty_generate(result.to_h)
+            File.write(File.join(TMP_DIR, "#{name}_result.json"), json)
+            puts json
+          end
+        end
+      end
+
+      private
+
+      def clone_if_needed(name, repo, ref)
+        dir = File.join(TMP_DIR, name)
+        return dir if Dir.exist?(File.join(dir, ".git"))
+
+        # Unified init + fetch + checkout for any ref (SHA / tag / branch).
+        # `git clone --branch` is noisier (annotated tags emit a "is not a
+        # commit" warning + detached HEAD advice) and doesn't accept SHAs.
+        # Fetching by SHA works thanks to GitHub's uploadpack.allowAnySHA1InWant.
+        system("git init -q #{dir}", exception: true)
+        system("git -C #{dir} remote add origin #{repo}", exception: true)
+        system("git -C #{dir} fetch --depth 1 -q origin #{ref}", exception: true)
+        system("git -C #{dir} checkout -q FETCH_HEAD", exception: true)
+        dir
+      end
+
+      def run_typeprof(name:, targets:, exclude:)
+        out_path = File.join(TMP_DIR, "#{name}_typeprof.out")
+        argv = ["-o", out_path, "--show-stats", *exclude.flat_map { ["--exclude", _1] }, *targets]
+
+        $stderr.puts "Running: typeprof #{argv.inspect}"
+        t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        system(TYPEPROF_BIN, *argv, exception: true)
+        elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t).round(2)
+
+        Result.new(name: name, elapsed: elapsed, **parse_and_compute(out_path))
+      end
+
+      def parse_and_compute(out_path)
+        # `--show-stats` appends the statistics block at the end of the same
+        # -o output file (after the RBS dump). Locate that block and parse it.
+        text = File.read(out_path)
+        idx = text.rindex("# TypeProf Evaluation Statistics") or raise "stats block not found in #{out_path}"
+        block = text[idx..]
+
+        methods = block[/Total methods:\s*(\d+)/, 1].to_i
+        fully_typed = block[/Fully typed:\s*(\d+)/, 1].to_i
+        overall_typed, overall_total = block.match(/Overall:\s*(\d+)\/(\d+)/).captures.map(&:to_i)
+
+        {
+          coverage_per_slot: (overall_typed * 100.0 / overall_total).round(2),
+          coverage_per_method: (fully_typed * 100.0 / methods).round(2),
+        }
+      end
+    end
+  end
+end

tool/benchmarks/optcarrot.rb

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require_relative "../benchmark_helper"
+
+TypeProf::Benchmark.run(
+  name: "optcarrot",
+  repo: "https://github.com/mame/optcarrot.git",
+  ref: "9c88f5f752341087270b0e86e741d73f19e52369", # 2026-04-29 HEAD
+)

tool/benchmarks/redmine.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+
+require_relative "../benchmark_helper"
+
+TypeProf::Benchmark.run(
+  name: "redmine",
+  repo: "https://github.com/redmine/redmine.git",
+  ref: "6.1.1",
+  targets: ["app", "sig"],
+  setup: -> {
+    # Each step is guarded with `unless ...` for idempotency: CI starts from a
+    # fresh clone, but local re-runs reuse tmp/redmine and skip done steps.
+    unless File.exist?("config/database.yml")
+      File.write("config/database.yml", <<~YAML)
+        development:
+          adapter: sqlite3
+          database: db/development.sqlite3
+      YAML
+    end
+    system("bundle add rbs_rails -v '0.13.1'", exception: true) unless File.read("Gemfile").include?("rbs_rails")
+    system("bin/rails db:migrate", exception: true) unless File.exist?("db/schema.rb")
+    system("bundle exec rbs collection init", exception: true) unless File.exist?("rbs_collection.yaml")
+    system("bundle exec rbs collection install", exception: true) unless File.exist?("rbs_collection.lock.yaml")
+    system("bundle exec rbs_rails all", exception: true) unless Dir.exist?("sig/rbs_rails")
+  },
+)

tool/benchmarks/typeprof.rb

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require_relative "../benchmark_helper"
+
+TypeProf::Benchmark.run(
+  name: "typeprof",
+  repo: "https://github.com/ruby/typeprof.git",
+  ref: "v0.31.1",
+  exclude: ["scenario/**/*"],
+)

tool/site/data.json

@@ -0,0 +1,3 @@
+{
+  "entries": []
+}

tool/site/index.html

@@ -0,0 +1,100 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>TypeProf Benchmarks</title>
+  <script src="https://cdn.jsdelivr.net/npm/chart.js@4"></script>
+  <style>
+    body { font-family: sans-serif; max-width: 1000px; margin: 0 auto; padding: 20px; }
+    header { margin-bottom: 16px; font-size: 0.9em; color: #555; }
+    header a { color: #36a2eb; text-decoration: none; }
+    header a:hover { text-decoration: underline; }
+    .project { margin-bottom: 40px; }
+    canvas { max-height: 300px; }
+  </style>
+</head>
+<body>
+<h1>TypeProf Benchmarks</h1>
+<header>
+  <strong>Last Update:</strong> <span id="last-update">-</span>
+  &nbsp;|&nbsp;
+  <strong>Repository:</strong> <a id="repo-link" rel="noopener">-</a>
+</header>
+<div id="charts"></div>
+<script>
+fetch('data.json').then(r => r.json()).then(data => {
+  const entries = data.entries;
+  const last = entries.at(-1)?.timestamp;
+  document.getElementById('last-update').textContent = last ? new Date(last).toLocaleString() : '-';
+  const link = document.getElementById('repo-link');
+  link.href = `https://github.com/${data.repo}`;
+  link.textContent = data.repo;
+
+  const labels = entries.map(e => e.sha.slice(0, 7));
+  const projects = [...new Set(entries.flatMap(e => e.projects.map(p => p.name)))];
+  const container = document.getElementById('charts');
+
+  const tooltipCallbacks = {
+    afterTitle: (items) => {
+      const e = entries[items[0].dataIndex];
+      return `${e.sha}\n${new Date(e.timestamp).toLocaleString()}`;
+    }
+  };
+  const onClick = (_event, elements) => {
+    if (!elements.length) return;
+    const e = entries[elements[0].index];
+    window.open(`https://github.com/${data.repo}/commit/${e.sha}`, '_blank');
+  };
+
+  const charts = [
+    {
+      id: 'elapsed',
+      title: 'elapsed (s)',
+      datasets: [{ key: 'elapsed', label: 'elapsed (s)', color: '#36a2eb' }],
+    },
+    {
+      id: 'type-coverage',
+      title: 'type coverage (%)',
+      yMax: 100,
+      // coverage_per_slot:   % of type slots (per argument/return) inferred
+      // coverage_per_method: % of methods whose all slots are inferred (fully typed)
+      datasets: [
+        { key: 'coverage_per_slot',   label: 'per slot',   color: '#4bc0c0' },
+        { key: 'coverage_per_method', label: 'per method', color: '#9966ff' },
+      ],
+    },
+  ];
+
+  for (const project of projects) {
+    const section = document.createElement('section');
+    section.className = 'project';
+    section.innerHTML = `<h2>${project}</h2>` +
+      charts.map(c => `<canvas id="${c.id}-${project}"></canvas>`).join('');
+    container.appendChild(section);
+
+    const projData = entries.map(e => e.projects.find(p => p.name === project));
+
+    for (const c of charts) {
+      const datasets = c.datasets.map(d => ({
+        label: d.label,
+        data: projData.map(p => p?.[d.key] ?? null),
+        borderColor: d.color,
+      }));
+      new Chart(document.getElementById(`${c.id}-${project}`), {
+        type: 'line',
+        data: { labels, datasets },
+        options: {
+          scales: { y: { beginAtZero: true, max: c.yMax } },
+          plugins: {
+            title: { display: true, text: c.title },
+            tooltip: { callbacks: tooltipCallbacks },
+          },
+          onClick,
+        },
+      });
+    }
+  }
+});
+</script>
+</body>
+</html>

tool/update_site_data.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+require "json"
+
+abort "Usage: ruby update_site_data.rb data.json result1.json [result2.json...]" if ARGV.size < 2
+data_path, *result_paths = ARGV
+
+sha = ENV.fetch("GITHUB_SHA") { `git rev-parse HEAD`.strip }
+
+data = JSON.parse(File.read(data_path))
+data["repo"] = ENV.fetch("GITHUB_REPOSITORY", "ruby/typeprof")
+data["entries"] << {
+  "sha" => sha,
+  "timestamp" => Time.now.iso8601,
+  "projects" => result_paths.map { JSON.parse(File.read(_1)) },
+}
+data["entries"] = data["entries"].last(500)
+
+json = JSON.pretty_generate(data)
+File.write(data_path, json)
+# Per-commit snapshot. The next bench run fetches its parent's snapshot
+# via this unique URL (<sha>.json), avoiding stale CDN responses that a
+# shared data.json URL would risk.
+File.write(File.join(File.dirname(data_path), "#{sha}.json"), json)

typeprof.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(.devcontainer|.github|scenario|sig|test|tool)/}) } - [ ".gitignore", "Gemfile", "Gemfile.lock", "Rakefile", "typeprof.conf.json"]
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(.devcontainer|.github|scenario|sig|test|tmp|tool)/}) } - [ ".gitignore", "Gemfile", "Gemfile.lock", "Rakefile", "typeprof.conf.json"]
   end
   spec.bindir        = "bin"
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }