[GR-65266] Add support for Shenandoah

compiler/ci/ci_common/benchmark-builders.jsonnet

@@ -81,6 +81,12 @@
   for suite in bench.groups.main_suites
   ],
 
+  local shenandoah_builds = [
+    c.weekly + hw.x52 + jdk + cc.libgraal + cc.shenandoah_mode + suite,
+  for jdk in cc.product_jdks
+  for suite in bench.groups.main_suites + [bench.specjbb2015]
+  ],
+
   local metrics_suites = [bench.dacapo, bench.scala_dacapo, bench.renaissance, bench.specjvm2008],
 
   local metrics_builds = std.flattenArrays([
@@ -92,7 +98,8 @@
   for suite in metrics_suites
   ]),
 
-  local all_builds = main_builds + weekly_amd64_forks_builds + weekly_aarch64_forks_builds + profiling_builds + avx_builds + zgc_builds + zgc_avx_builds + aarch64_builds + metrics_builds,
+  local all_builds = main_builds + weekly_amd64_forks_builds + weekly_aarch64_forks_builds + profiling_builds + avx_builds + zgc_builds + zgc_avx_builds +
+                     shenandoah_builds + aarch64_builds + metrics_builds,
   local filtered_builds = [b for b in all_builds if b.is_jdk_supported(b.jdk_version) && b.is_arch_supported(b.arch)],
   // adds a "defined_in" field to all builds mentioning the location of this current file
   builds:: utils.add_defined_in(filtered_builds, std.thisFile),

compiler/ci/ci_common/compiler-common.libsonnet

@@ -191,6 +191,13 @@
     }
   },
 
+  shenandoah_mode:: {
+    platform+:: "-shenandoah",
+    environment+: {
+      "JVM_CONFIG"+: "-shenandoah",
+    }
+  },
+
   serialgc_mode:: {
     platform+:: "-serialgc",
     environment+: {

compiler/ci/ci_common/gate.jsonnet

@@ -101,9 +101,15 @@
   test:: s.base(no_warning_as_error=true, extra_vm_args="-Djdk.graal.DetailedAsserts=true"),
   unittest_compiler:: s.base(tags="build,unittest-compiler", no_warning_as_error=true, extra_vm_args="-Djdk.graal.DetailedAsserts=true"),
   unittest_truffle:: s.base(tags="build,unittest-truffle", no_warning_as_error=true, extra_vm_args="-Djdk.graal.DetailedAsserts=true"),
+
   test_zgc:: s.base(no_warning_as_error=true, extra_vm_args="-XX:+UseZGC"),
   unittest_compiler_zgc:: s.base(tags="build,unittest-compiler", no_warning_as_error=true, extra_vm_args="-XX:+UseZGC"),
   unittest_truffle_zgc:: s.base(tags="build,unittest-truffle", no_warning_as_error=true, extra_vm_args="-XX:+UseZGC"),
+
+  test_shenandoah:: s.base(no_warning_as_error=true, extra_vm_args="-XX:+UseShenandoahGC"),
+  unittest_compiler_shenandoah:: s.base(tags="build,unittest-compiler", no_warning_as_error=true, extra_vm_args="-XX:+UseShenandoahGC"),
+  unittest_truffle_shenandoah:: s.base(tags="build,unittest-truffle", no_warning_as_error=true, extra_vm_args="-XX:+UseShenandoahGC"),
+
   test_serialgc:: s.base(no_warning_as_error=true, extra_vm_args="-XX:+UseSerialGC"),
 
   jacoco_gate_args:: ["--jacoco-omit-excluded", "--jacoco-relativize-paths", "--jacoco-omit-src-gen", "--jacocout", "coverage", "--jacoco-format", "lcov"],
@@ -131,55 +137,39 @@
 
   # Runs truffle tests in a mode similar to HotSpot's -Xcomp option
   # (i.e. compile immediately without background compilation).
-  truffle_xcomp:: s.base("build,unittest",
+  truffle_xcomp_base(extra_vm_args=""):: s.base("build,unittest",
     extra_vm_args="-Dpolyglot.engine.AllowExperimentalOptions=true " +
                   "-Dpolyglot.engine.CompileImmediately=true " +
                   "-Dpolyglot.engine.BackgroundCompilation=false " +
-                  "-Dtck.inlineVerifierInstrument=false",
+                  "-Dtck.inlineVerifierInstrument=false" + extra_vm_args,
     extra_unittest_args="--verbose truffle") + {
       environment+: {"TRACE_COMPILATION": "true"},
       logs+: ["*/*_compilation.log"],
       components+: ["truffle"],
     },
 
-  truffle_xcomp_zgc:: s.base("build,unittest",
-    extra_vm_args="-Dpolyglot.engine.AllowExperimentalOptions=true " +
-                  "-Dpolyglot.engine.CompileImmediately=true " +
-                  "-Dpolyglot.engine.BackgroundCompilation=false " +
-                  "-Dtck.inlineVerifierInstrument=false " +
-                  "-XX:+UseZGC",
-    extra_unittest_args="--verbose truffle") + {
-      environment+: {"TRACE_COMPILATION": "true"},
-      logs+: ["*/*_compilation.log"],
-      components+: ["truffle"],
-    },
-
-  truffle_xcomp_serialgc:: s.base("build,unittest",
-    extra_vm_args="-Dpolyglot.engine.AllowExperimentalOptions=true " +
-                  "-Dpolyglot.engine.CompileImmediately=true " +
-                  "-Dpolyglot.engine.BackgroundCompilation=false " +
-                  "-Dtck.inlineVerifierInstrument=false " +
-                  "-XX:+UseSerialGC",
-    extra_unittest_args="--verbose truffle") + {
-      environment+: {"TRACE_COMPILATION": "true"},
-      logs+: ["*/*_compilation.log"],
-      components+: ["truffle"],
-    },
+  truffle_xcomp:: s.truffle_xcomp_base(),
+  truffle_xcomp_zgc:: s.truffle_xcomp_base(" -XX:+UseZGC"),
+  truffle_xcomp_shenandoah:: s.truffle_xcomp_base(" -XX:+UseShenandoahGC"),
+  truffle_xcomp_serialgc:: s.truffle_xcomp_base(" -XX:+UseSerialGC"),
 
   ctw:: s.base("build,ctw", no_warning_as_error=true),
   ctw_zgc:: s.base("build,ctw", no_warning_as_error=true, extra_vm_args="-XX:+UseZGC"),
+  ctw_shenandoah:: s.base("build,ctw", no_warning_as_error=true, extra_vm_args="-XX:+UseShenandoahGC"),
 
   ctw_economy:: s.base("build,ctweconomy", extra_vm_args="-Djdk.graal.CompilerConfiguration=economy"),
   ctw_phaseplan_fuzzing:: s.base("build,ctwphaseplanfuzzing"),
 
   # Runs some benchmarks as tests
   benchmarktest:: s.base("build,benchmarktest") + jmh_benchmark_test,
   benchmarktest_zgc:: s.base("build,benchmarktest", extra_vm_args="-XX:+UseZGC") + jmh_benchmark_test,
+  benchmarktest_shenandoah:: s.base("build,benchmarktest", extra_vm_args="-XX:+UseShenandoahGC") + jmh_benchmark_test,
 
   bootstrap:: s.base("build,bootstrap", no_warning_as_error=true),
   bootstrap_lite:: s.base("build,bootstraplite", no_warning_as_error=true),
   bootstrap_full:: s.base("build,bootstrapfullverify", no_warning_as_error=true),
   bootstrap_full_zgc:: s.base("build,bootstrapfullverify", no_warning_as_error=true, extra_vm_args="-XX:+UseZGC"),
+  bootstrap_full_shenandoah:: s.base("build,bootstrapfullverify", no_warning_as_error=true, extra_vm_args="-XX:+UseShenandoahGC"),
   bootstrap_economy:: s.base("build,bootstrapeconomy", no_warning_as_error=true, extra_vm_args="-Djdk.graal.CompilerConfiguration=economy"),
 
   style:: c.deps.eclipse + c.deps.jdt + c.deps.spotbugs + s.base("style,fullbuild,javadoc") + galahad.exclude,
@@ -491,28 +481,45 @@
      self.make_build(self.jdk_latest, "linux-amd64", "coverage_avx3").build
   ],
 
-    # Test ZGC on supported platforms.  Windows requires version 1083 or later which will
-    # probably require adding some capabilities.
-    local all_zgc_builds = [self.make_build(jdk, os_arch, task).build + galahad.exclude
-      for jdk in [
-        self.jdk_latest
-      ]
-      for os_arch in [
-        "linux-amd64",
-        "linux-aarch64",
-        "darwin-amd64",
-        "darwin-aarch64"
-      ]
-      for task in [
-        "test_zgc",
-        "unittest_compiler_zgc",
-        "unittest_truffle_zgc",
-        "truffle_xcomp_zgc",
-        "ctw_zgc",
-        "benchmarktest_zgc",
-        "bootstrap_full_zgc",
-      ]
-    ],
+  # Test ZGC on supported platforms.  Windows requires version 1083 or later which will
+  # probably require adding some capabilities.
+  local all_zgc_builds = [self.make_build(jdk, os_arch, task).build + galahad.exclude
+    for jdk in [
+      self.jdk_latest
+    ]
+    for os_arch in [
+      "linux-amd64",
+      "linux-aarch64",
+      "darwin-amd64",
+      "darwin-aarch64"
+    ]
+    for task in [
+      "test_zgc",
+      "unittest_compiler_zgc",
+      "unittest_truffle_zgc",
+      "truffle_xcomp_zgc",
+      "ctw_zgc",
+      "benchmarktest_zgc",
+      "bootstrap_full_zgc",
+    ]
+  ],
+
+  # Test Shenandoah on supported platforms.
+  local all_shenandoah_builds = [self.make_build(jdk, os_arch, task).build + galahad.exclude
+    for jdk in [
+      self.jdk_latest
+    ]
+    for os_arch in all_os_arches
+    for task in [
+      "test_shenandoah",
+      "unittest_compiler_shenandoah",
+      "unittest_truffle_shenandoah",
+      "truffle_xcomp_shenandoah",
+      "ctw_shenandoah",
+      "benchmarktest_shenandoah",
+      "bootstrap_full_shenandoah",
+    ]
+  ],
 
   # Run unittests with SerialGC.
   local all_serialgc_builds = [self.make_build(self.jdk_latest, os_arch, task).build + galahad.exclude
@@ -563,6 +570,7 @@
     all_platforms_builds +
     all_coverage_builds +
     all_zgc_builds +
+    all_shenandoah_builds +
     all_serialgc_builds +
     style_builds +
     linux_amd64_jdk_latest_builds +

compiler/ci/ci_includes/baseline-benchmarks.jsonnet

@@ -60,6 +60,7 @@
   local gc_variants_builds = std.flattenArrays([
     [
     c.monthly + hw.x52 + jdk + cc.c2                         + cc.zgc_mode + suite,
+    c.monthly + hw.x52 + jdk + cc.c2                         + cc.shenandoah_mode + suite,
     ]
   for jdk in cc.product_jdks
   for suite in bench.groups.main_suites
@@ -68,6 +69,7 @@
     c.monthly + hw.x52 + jdk + cc.c2                         + cc.serialgc_mode + bench.microservice_benchmarks,
     c.monthly + hw.x52 + jdk + cc.c2                         + cc.pargc_mode    + bench.microservice_benchmarks,
     c.monthly + hw.x52 + jdk + cc.c2                         + cc.zgc_mode      + bench.microservice_benchmarks,
+    c.monthly + hw.x52 + jdk + cc.c2                         + cc.shenandoah_mode      + bench.microservice_benchmarks,
     ]
   for jdk in cc.product_jdks
   ]),

compiler/mx.compiler/mx_graal_benchmark.py

@@ -48,6 +48,7 @@
     ('zgc', ['-XX:+UseZGC'], 12),
     ('zgc-avx2', ['-XX:+UseZGC', '-XX:UseAVX=2'], 12),
     ('zgc-avx3', ['-XX:+UseZGC', '-XX:UseAVX=3'], 12),
+    ('shenandoah', ['-XX:+UseShenandoahGC'], 12),
     ('no-comp-oops', ['-XX:-UseCompressedOops'], 0),
     ('no-profile-info', ['-Djvmci.UseProfilingInformation=false'], 0),
     ('no-splitting', ['-Dpolyglot.engine.Splitting=false'], 0),

compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/replacements/test/PointerTrackingTest.java

@@ -24,23 +24,33 @@
  */
 package jdk.graal.compiler.replacements.test;
 
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+
 import jdk.graal.compiler.api.directives.GraalDirectives;
+import jdk.graal.compiler.api.test.Graal;
 import jdk.graal.compiler.debug.DebugCloseable;
 import jdk.graal.compiler.debug.DebugContext;
 import jdk.graal.compiler.debug.GraalError;
+import jdk.graal.compiler.hotspot.HotSpotGraalRuntime;
 import jdk.graal.compiler.nodes.ValueNode;
 import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext;
 import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugin;
 import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugins;
 import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugins.Registration;
 import jdk.graal.compiler.replacements.Snippets;
+import jdk.graal.compiler.runtime.RuntimeProvider;
 import jdk.graal.compiler.word.WordCastNode;
-import org.junit.Test;
-
 import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 
 public class PointerTrackingTest extends ReplacementsTest implements Snippets {
+    @Before
+    public void before() {
+        HotSpotGraalRuntime runtime = (HotSpotGraalRuntime) Graal.getRequiredCapability(RuntimeProvider.class);
+        Assume.assumeTrue("doesn't aggressively move objects", runtime.getGarbageCollector() != HotSpotGraalRuntime.HotSpotGC.Shenandoah);
+    }
 
     @Test
     public void testTracking() {

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/common/GraalOptions.java

@@ -193,6 +193,8 @@ public final class GraalOptions {
     // Register allocator debugging
     @Option(help = "Comma separated list of registers that register allocation is limited to.", type = OptionType.Debug)
     public static final OptionKey<String> RegisterPressure = new OptionKey<>(null);
+    @Option(help = "Permit RegisterPressure setting to cause compilation to fail.", type = OptionType.Debug)
+    public static final OptionKey<Boolean> BailoutOnRegisterPressureFailure = new OptionKey<>(false);
 
     @Option(help = "Eliminates redundant conditional expressions and statements where possible. " +
                    "This can improve performance because fewer logic instructions have to be executed.", type = OptionType.Expert)

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/gen/LIRCompilerBackend.java

@@ -24,6 +24,7 @@
  */
 package jdk.graal.compiler.core.gen;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 
@@ -114,12 +115,12 @@ public static LIRGenerationResult emitLIR(Backend backend, StructuredGraph graph
         try {
             return emitLIR0(backend, graph, stub, registerConfig, lirSuites, allocationRestrictedTo, entryPointDecorator);
         } catch (OutOfRegistersException e) {
-            if (allocationRestrictedTo != null) {
+            if (allocationRestrictedTo != null && !GraalOptions.BailoutOnRegisterPressureFailure.getValue(graph.getOptions())) {
                 allocationRestrictedTo = null;
                 return emitLIR0(backend, graph, stub, registerConfig, lirSuites, allocationRestrictedTo, entryPointDecorator);
             }
             /* If the re-execution fails we convert the exception into a "hard" failure */
-            throw new GraalError(e);
+            throw new GraalError(e, "out of registers%s", allocationRestrictedTo == null ? "" : ": " + Arrays.toString(allocationRestrictedTo));
         } finally {
             graph.checkCancellation();
         }

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyLowTier.java

@@ -26,6 +26,7 @@
 
 import jdk.graal.compiler.debug.Assertions;
 import jdk.graal.compiler.graph.Graph;
+import jdk.graal.compiler.nodes.GraphState;
 import jdk.graal.compiler.options.OptionValues;
 import jdk.graal.compiler.phases.PlaceholderPhase;
 import jdk.graal.compiler.phases.common.AddressLoweringPhase;
@@ -37,6 +38,7 @@
 import jdk.graal.compiler.phases.common.LowTierLoweringPhase;
 import jdk.graal.compiler.phases.common.RemoveOpaqueValuePhase;
 import jdk.graal.compiler.phases.common.TransplantGraphsPhase;
+import jdk.graal.compiler.phases.common.WriteBarrierAdditionPhase;
 import jdk.graal.compiler.phases.schedule.SchedulePhase;
 import jdk.graal.compiler.phases.tiers.LowTierContext;
 
@@ -51,6 +53,7 @@ public EconomyLowTier(OptionValues options) {
         appendPhase(new LowTierLoweringPhase(canonicalizer));
         appendPhase(new ExpandLogicPhase(canonicalizer));
 
+        appendPhase(new WriteBarrierAdditionPhase(GraphState.StageFlag.LOW_TIER_BARRIER_ADDITION));
         appendPhase(new EconomyPiRemovalPhase(canonicalizer));
 
         if (Assertions.assertionsEnabled()) {

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java

@@ -28,6 +28,7 @@
 
 import jdk.graal.compiler.core.common.GraalOptions;
 import jdk.graal.compiler.graph.Graph;
+import jdk.graal.compiler.nodes.GraphState;
 import jdk.graal.compiler.options.Option;
 import jdk.graal.compiler.options.OptionKey;
 import jdk.graal.compiler.options.OptionType;
@@ -47,6 +48,7 @@
 import jdk.graal.compiler.phases.common.PropagateDeoptimizeProbabilityPhase;
 import jdk.graal.compiler.phases.common.RemoveOpaqueValuePhase;
 import jdk.graal.compiler.phases.common.TransplantGraphsPhase;
+import jdk.graal.compiler.phases.common.WriteBarrierAdditionPhase;
 import jdk.graal.compiler.phases.schedule.SchedulePhase;
 import jdk.graal.compiler.phases.schedule.SchedulePhase.SchedulingStrategy;
 import jdk.graal.compiler.phases.tiers.LowTierContext;
@@ -87,6 +89,8 @@ public LowTier(OptionValues options) {
         appendPhase(new FixReadsPhase(true,
                         new SchedulePhase(GraalOptions.StressTestEarlyReads.getValue(options) ? SchedulingStrategy.EARLIEST : SchedulingStrategy.LATEST_OUT_OF_LOOPS_IMPLICIT_NULL_CHECKS)));
 
+        appendPhase(new WriteBarrierAdditionPhase(GraphState.StageFlag.LOW_TIER_BARRIER_ADDITION));
+
         appendPhase(canonicalizerWithoutGVN);
 
         /*

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java

@@ -612,6 +612,44 @@ private long getZGCAddressField(String name) {
     public final long zBarrierSetRuntimeLoadBarrierOnOopArray = getZGCAddressField("ZBarrierSetRuntime::load_barrier_on_oop_array");
     public final int zPointerLoadShift = getConstant("ZPointerLoadShift", Integer.class, -1, osArch.equals("aarch64"));
 
+    /*
+     * Shenandoah GC support.
+     */
+    /**
+     * Indicates whether or not the HotSpot VM has been built with Shenandoah support. If not, then
+     * we don't expect the Shenandoah symbols to be present in JVMCI.
+     */
+    public final boolean hasShenandoahGC = getStore().getConstants().containsKey("INCLUDE_SHENANDOAHGC") && getConstant("INCLUDE_SHENANDOAHGC", Boolean.class);
+
+    /*
+     * Various Shenandoah GC constants.
+     */
+    public final int shenandoahGCStateOffset = getConstant("ShenandoahThreadLocalData::gc_state_offset", Integer.class, -1, hasShenandoahGC);
+    public final int shenandoahSATBIndexOffset = getConstant("ShenandoahThreadLocalData::satb_mark_queue_index_offset", Integer.class, -1, hasShenandoahGC);
+    public final int shenandoahSATBBufferOffset = getConstant("ShenandoahThreadLocalData::satb_mark_queue_buffer_offset", Integer.class, -1, hasShenandoahGC);
+    public final int shenandoahCardTableOffset = getConstant("ShenandoahThreadLocalData::card_table_offset", Integer.class, -1, hasShenandoahGC);
+    public final int shenandoahGCRegionSizeBytesShift = getFieldValue("CompilerToVM::Data::shenandoah_region_size_bytes_shift", Integer.class, "int", -1, hasShenandoahGC);
+    public final long shenandoahGCCSetFastTestAddress = getFieldValue("CompilerToVM::Data::shenandoah_in_cset_fast_test_addr", Long.class, "address", -1L, hasShenandoahGC);
+
+    public final int shenandoahGCStateHasForwarded = getConstant("ShenandoahHeap::HAS_FORWARDED", Integer.class, -1, hasShenandoahGC);
+    public final int shenandoahGCStateMarking = getConstant("ShenandoahHeap::MARKING", Integer.class, -1, hasShenandoahGC);
+    public final int shenandoahGCStateEvacuation = getConstant("ShenandoahHeap::EVACUATION", Integer.class, -1, hasShenandoahGC);
+    public final int shenandoahGCStateUpdateRefs = getConstant("ShenandoahHeap::UPDATE_REFS", Integer.class, -1, hasShenandoahGC);
+    public final int shenandoahGCStateWeakRoots = getConstant("ShenandoahHeap::WEAK_ROOTS", Integer.class, -1, hasShenandoahGC);
+    public final int shenandoahGCStateYoungMarking = getConstant("ShenandoahHeap::YOUNG_MARKING", Integer.class, -1, hasShenandoahGC);
+    public final int shenandoahGCStateOldMarking = getConstant("ShenandoahHeap::OLD_MARKING", Integer.class, -1, hasShenandoahGC);
+
+    /*
+     * Shenandoah barrier slow-paths.
+     */
+    public final long shenandoahLoadBarrierStrong = getAddress("ShenandoahRuntime::load_reference_barrier_strong", -1L, hasShenandoahGC);
+    public final long shenandoahLoadBarrierStrongNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_strong_narrow", -1L, hasShenandoahGC);
+    public final long shenandoahLoadBarrierWeak = getAddress("ShenandoahRuntime::load_reference_barrier_weak", -1L, hasShenandoahGC);
+    public final long shenandoahLoadBarrierWeakNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_weak_narrow", -1L, hasShenandoahGC);
+    public final long shenandoahLoadBarrierPhantom = getAddress("ShenandoahRuntime::load_reference_barrier_phantom", -1L, hasShenandoahGC);
+    public final long shenandoahLoadBarrierPhantomNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_phantom_narrow", -1L, hasShenandoahGC);
+    public final long shenandoahWriteBarrierPre = getAddress("ShenandoahRuntime::write_barrier_pre", -1L, hasShenandoahGC);
+
     // aarch64 specific nmethod entry barrier support
     // @formatter:off
     public final int BarrierSetAssembler_nmethod_patching_type = getFieldValue("CompilerToVM::Data::BarrierSetAssembler_nmethod_patching_type", Integer.class, "int", -1, osArch.equals("aarch64"));

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java

@@ -292,6 +292,8 @@ private BarrierSet createBarrierSet(GraalHotSpotVMConfig config, MetaAccessProvi
         ResolvedJavaField referentField = HotSpotReplacementsUtil.referentField(metaAccess);
         if (config.gc == HotSpotGraalRuntime.HotSpotGC.Z) {
             return new HotSpotZBarrierSet(objectArrayType, referentField);
+        } else if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) {
+            return new HotSpotShenandoahBarrierSet(objectArrayType, referentField, config);
         } else if (config.gc == HotSpotGraalRuntime.HotSpotGC.Epsilon) {
             return new NoBarrierSet();
         } else if (config.useG1GC()) {