diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index ad25c07f293..fcbc5f514c5 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -15,7 +15,7 @@ on: jobs: lint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: quay.io/cortexproject/build-image:master-582c03a76 steps: @@ -44,7 +44,7 @@ jobs: run: make BUILD_IN_CONTAINER=false check-white-noise test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: quay.io/cortexproject/build-image:master-582c03a76 steps: @@ -64,7 +64,7 @@ jobs: security: name: CodeQL - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 permissions: actions: read contents: read @@ -87,7 +87,7 @@ jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: quay.io/cortexproject/build-image:master-582c03a76 steps: @@ -132,7 +132,7 @@ jobs: integration: needs: build - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -206,7 +206,7 @@ jobs: integration-configs-db: needs: build - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout Repo uses: actions/checkout@v2 @@ -228,7 +228,7 @@ jobs: deploy_website: needs: [build, test] if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex' - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: quay.io/cortexproject/build-image:master-582c03a76 steps: @@ -270,7 +270,7 @@ jobs: deploy: needs: [build, test, lint, integration, integration-configs-db] if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex' - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: quay.io/cortexproject/build-image:master-582c03a76 steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b53f6de9a9..ed9f914fdb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## master / unreleased +## 1.18.2 2025-07-09 + +* [BUGFIX] Backporting Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled #6271 + ## 1.18.1 2024-10-14 * [BUGFIX] Backporting upgrade to go 1.22.7 to patch CVE-2024-34155, CVE-2024-34156, CVE-2024-34158 #6217 #6264 diff --git a/VERSION b/VERSION index ec6d649be65..b57fc7228b6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.18.1 +1.18.2 diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index d4f1e5735b7..2ae6c34caf6 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -684,6 +684,9 @@ func (i *Lifecycler) initRing(ctx context.Context) error { level.Info(i.logger).Log("msg", "existing entry found in ring", "state", i.GetState(), "tokens", len(tokens), "ring", i.RingName) + // Update the address if it has changed + instanceDesc.Addr = i.Addr + // Update the ring if the instance has been changed and the heartbeat is disabled. // We dont need to update KV here when heartbeat is enabled as this info will eventually be update on KV // on the next heartbeat diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go index 0b8a6402db3..b121ef258a9 100644 --- a/pkg/ring/lifecycler_test.go +++ b/pkg/ring/lifecycler_test.go @@ -40,6 +40,14 @@ func testLifecyclerConfig(ringConfig Config, id string) LifecyclerConfig { return lifecyclerConfig } +// testLifecyclerConfigWithAddr creates a LifecyclerConfig with the given address. +// This is useful for testing when we want to set the address to a specific value. +func testLifecyclerConfigWithAddr(ringConfig Config, id string, addr string) LifecyclerConfig { + l := testLifecyclerConfig(ringConfig, id) + l.Addr = addr + return l +} + func checkNormalised(d interface{}, id string) bool { desc, ok := d.(*Desc) return ok && @@ -644,8 +652,8 @@ func TestRestartIngester_DisabledHeartbeat_unregister_on_shutdown_false(t *testi } // Starts Ingester and wait it to became active - startIngesterAndWaitActive := func(ingId string) *Lifecycler { - lifecyclerConfig := testLifecyclerConfig(ringConfig, ingId) + startIngesterAndWaitActive := func(ingId string, addr string) *Lifecycler { + lifecyclerConfig := testLifecyclerConfigWithAddr(ringConfig, ingId, addr) // Disabling heartBeat and unregister_on_shutdown lifecyclerConfig.UnregisterOnShutdown = false lifecyclerConfig.HeartbeatPeriod = 0 @@ -662,10 +670,10 @@ func TestRestartIngester_DisabledHeartbeat_unregister_on_shutdown_false(t *testi // test if the ingester 2 became active after: // * Clean Shutdown (LEAVING after shutdown) // * Crashes while in the PENDING or JOINING state - l1 := startIngesterAndWaitActive("ing1") + l1 := startIngesterAndWaitActive("ing1", "0.0.0.0") defer services.StopAndAwaitTerminated(context.Background(), l1) //nolint:errcheck - l2 := startIngesterAndWaitActive("ing2") + l2 := startIngesterAndWaitActive("ing2", "0.0.0.0") ingesters := poll(func(desc *Desc) bool { return len(desc.Ingesters) == 2 && desc.Ingesters["ing1"].State == ACTIVE && desc.Ingesters["ing2"].State == ACTIVE @@ -684,7 +692,7 @@ func TestRestartIngester_DisabledHeartbeat_unregister_on_shutdown_false(t *testi assert.Equal(t, LEAVING, ingesters["ing2"].State) // Start Ingester2 again - Should flip back to ACTIVE in the ring - l2 = startIngesterAndWaitActive("ing2") + l2 = startIngesterAndWaitActive("ing2", "0.0.0.0") require.NoError(t, services.StopAndAwaitTerminated(context.Background(), l2)) // Simulate ingester2 crash on startup and left the ring with JOINING state @@ -698,7 +706,7 @@ func TestRestartIngester_DisabledHeartbeat_unregister_on_shutdown_false(t *testi }) require.NoError(t, err) - l2 = startIngesterAndWaitActive("ing2") + l2 = startIngesterAndWaitActive("ing2", "0.0.0.0") require.NoError(t, services.StopAndAwaitTerminated(context.Background(), l2)) // Simulate ingester2 crash on startup and left the ring with PENDING state @@ -712,7 +720,26 @@ func TestRestartIngester_DisabledHeartbeat_unregister_on_shutdown_false(t *testi }) require.NoError(t, err) - l2 = startIngesterAndWaitActive("ing2") + l2 = startIngesterAndWaitActive("ing2", "0.0.0.0") + require.NoError(t, services.StopAndAwaitTerminated(context.Background(), l2)) + + // Simulate ingester2 crashing and left the ring with ACTIVE state, but when it comes up + // it has a different ip address + startIngesterAndWaitActive("ing2", "0.0.0.0") + ingesters = poll(func(desc *Desc) bool { + return desc.Ingesters["ing2"].State == ACTIVE && desc.Ingesters["ing2"].Addr == "0.0.0.0:1" + }) + assert.Equal(t, ACTIVE, ingesters["ing2"].State) + assert.Equal(t, "0.0.0.0:1", ingesters["ing2"].Addr) + + l2 = startIngesterAndWaitActive("ing2", "1.1.1.1") + + // The ring should have the new ip address + ingesters = poll(func(desc *Desc) bool { + return desc.Ingesters["ing2"].State == ACTIVE && desc.Ingesters["ing2"].Addr == "1.1.1.1:1" + }) + assert.Equal(t, ACTIVE, ingesters["ing2"].State) + assert.Equal(t, "1.1.1.1:1", ingesters["ing2"].Addr) require.NoError(t, services.StopAndAwaitTerminated(context.Background(), l2)) }