fatal error: concurrent map writes

[edwloef]

fatal error: concurrent map writes

goroutine 267 [running]:
internal/runtime/maps.fatal({0x9bdf2e?, 0xc000100008?})
	/usr/lib/go/src/runtime/panic.go:1058 +0x18
net/textproto.MIMEHeader.Add(...)
	/usr/lib/go/src/net/textproto/header.go:15
net/http.Header.Add(...)
	/usr/lib/go/src/net/http/header.go:31
github.com/labstack/echo/v4/middleware.CORSWithConfig.func1.1({0xa7f210, 0xc0000cdc70})
	/home/may/.go/pkg/mod/github.com/labstack/echo/[email protected]/middleware/cors.go:191 +0x1b4
github.com/labstack/echo/v4/middleware.BodyLimitWithConfig.func1.1({0xa7f210, 0xc0000cdc70})
	/home/may/.go/pkg/mod/github.com/labstack/echo/[email protected]/middleware/body_limit.go:85 +0x15c
main.main.func1.1({0xa7f3e0?, 0xc0000c9040?})
	/home/may/uni/swep/backend/main.go:45 +0x59
github.com/labstack/echo/v4.(*Echo).ServeHTTP(0xc00019a488, {0xa752c0, 0xc0004420e0}, 0xc000306500)
	/home/may/.go/pkg/mod/github.com/labstack/echo/[email protected]/echo.go:668 +0x323
net/http.serverHandler.ServeHTTP({0xc000302c60?}, {0xa752c0?, 0xc0004420e0?}, 0x6?)
	/usr/lib/go/src/net/http/server.go:3301 +0x8e
net/http.(*conn).serve(0xc000157680, {0xa76400, 0xc0001bd1d0})
	/usr/lib/go/src/net/http/server.go:2102 +0x625
created by net/http.(*Server).Serve in goroutine 1
	/usr/lib/go/src/net/http/server.go:3454 +0x485

goroutine 1 [IO wait]:
internal/poll.runtime_pollWait(0x7f6de8c7f830, 0x72)
	/usr/lib/go/src/runtime/netpoll.go:351 +0x85
internal/poll.(*pollDesc).wait(0xc000052d00?, 0xc000002380?, 0x0)
	/usr/lib/go/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/usr/lib/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000052d00)
	/usr/lib/go/src/internal/poll/fd_unix.go:620 +0x295
net.(*netFD).accept(0xc000052d00)
	/usr/lib/go/src/net/fd_unix.go:172 +0x29
net.(*TCPListener).accept(0xc0000aef40)
	/usr/lib/go/src/net/tcpsock_posix.go:159 +0x1b
net.(*TCPListener).AcceptTCP(0xc0000aef40)
	/usr/lib/go/src/net/tcpsock.go:367 +0x30
github.com/labstack/echo/v4.tcpKeepAliveListener.Accept({0x448c40?})
	/home/may/.go/pkg/mod/github.com/labstack/echo/[email protected]/echo.go:988 +0x17
net/http.(*Server).Serve(0xc0001b4200, {0xa75950, 0xc00007e260})
	/usr/lib/go/src/net/http/server.go:3424 +0x30c
github.com/labstack/echo/v4.(*Echo).Start(0xc00019a488, {0x9b31f0, 0x5})
	/home/may/.go/pkg/mod/github.com/labstack/echo/[email protected]/echo.go:685 +0xcb
main.main()
	/home/may/uni/swep/backend/main.go:66 +0xe52

goroutine 7 [select]:
github.com/jackc/pgx/v5/pgxpool.(*Pool).backgroundHealthCheck(0xc0000e6180)
	/home/may/.go/pkg/mod/github.com/jackc/pgx/[email protected]/pgxpool/pool.go:433 +0xc5
github.com/jackc/pgx/v5/pgxpool.NewWithConfig.func3()
	/home/may/.go/pkg/mod/github.com/jackc/pgx/[email protected]/pgxpool/pool.go:286 +0x3d
created by github.com/jackc/pgx/v5/pgxpool.NewWithConfig in goroutine 1
	/home/may/.go/pkg/mod/github.com/jackc/pgx/[email protected]/pgxpool/pool.go:283 +0x37c

goroutine 308 [runnable]:
net/http.(*connReader).startBackgroundRead.gowrap2()
	/usr/lib/go/src/net/http/server.go:686
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1
created by net/http.(*connReader).startBackgroundRead in goroutine 266
	/usr/lib/go/src/net/http/server.go:686 +0xb6

goroutine 257 [IO wait]:
internal/poll.runtime_pollWait(0x7f6de8c7f2b8, 0x72)
	/usr/lib/go/src/runtime/netpoll.go:351 +0x85
internal/poll.(*pollDesc).wait(0xc000211900?, 0xc0001e9000?, 0x0)
	/usr/lib/go/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/usr/lib/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000211900, {0xc0001e9000, 0x1000, 0x1000})
	/usr/lib/go/src/internal/poll/fd_unix.go:165 +0x27a
net.(*netFD).Read(0xc000211900, {0xc0001e9000?, 0xc0001a9ad0?, 0x4cea05?})
	/usr/lib/go/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc000212520, {0xc0001e9000?, 0x0?, 0x0?})
	/usr/lib/go/src/net/net.go:194 +0x45
net/http.(*connReader).Read(0xc00031d290, {0xc0001e9000, 0x1000, 0x1000})
	/usr/lib/go/src/net/http/server.go:798 +0x159
bufio.(*Reader).fill(0xc0000b09c0)
	/usr/lib/go/src/bufio/bufio.go:113 +0x103
bufio.(*Reader).Peek(0xc0000b09c0, 0x4)
	/usr/lib/go/src/bufio/bufio.go:152 +0x53
net/http.(*conn).serve(0xc000215ef0, {0xa76400, 0xc0001bd1d0})
	/usr/lib/go/src/net/http/server.go:2137 +0x785
created by net/http.(*Server).Serve in goroutine 1
	/usr/lib/go/src/net/http/server.go:3454 +0x485

goroutine 266 [runnable]:
main.(*LiveRelationsConfig).Get(0xc0000cdce0, {0xa7de70, 0xc0000f43f0})
	/home/may/uni/swep/backend/relations.go:24 +0x5f6
main.GetRelations({0xa7f210, 0xc0000cdc70})
	/home/may/uni/swep/backend/endpoints.go:18 +0x75
github.com/labstack/echo/v4.(*Echo).add.func1({0xa7f210, 0xc0000cdc70})
	/home/may/.go/pkg/mod/github.com/labstack/echo/[email protected]/echo.go:581 +0x45
main.main.Secure.SecureWithConfig.func4.1({0xa7f210, 0xc0000cdc70})
	/home/may/.go/pkg/mod/github.com/labstack/echo/[email protected]/middleware/secure.go:141 +0x364
main.main.Recover.RecoverWithConfig.func3.1({0xa7f210, 0xc0000cdc70})
	/home/may/.go/pkg/mod/github.com/labstack/echo/[email protected]/middleware/recover.go:130 +0x111
main.main.RateLimiter.RateLimiterWithConfig.func2.1({0xa7f210, 0xc0000cdc70})
	/home/may/.go/pkg/mod/github.com/labstack/echo/[email protected]/middleware/rate_limiter.go:146 +0xba
github.com/labstack/echo/v4/middleware.LoggerWithConfig.func2.1({0xa7f210, 0xc0000cdc70})
	/home/may/.go/pkg/mod/github.com/labstack/echo/[email protected]/middleware/logger.go:125 +0xd8
github.com/labstack/echo/v4/middleware.CORSWithConfig.func1.1({0xa7f210, 0xc0000cdc70})
	/home/may/.go/pkg/mod/github.com/labstack/echo/[email protected]/middleware/cors.go:280 +0x8a2
github.com/labstack/echo/v4/middleware.BodyLimitWithConfig.func1.1({0xa7f210, 0xc0000cdc70})
	/home/may/.go/pkg/mod/github.com/labstack/echo/[email protected]/middleware/body_limit.go:85 +0x15c
main.main.func1.1({0xa7f3e0?, 0xc000434000?})
	/home/may/uni/swep/backend/main.go:45 +0x59
github.com/labstack/echo/v4.(*Echo).ServeHTTP(0xc00019a488, {0xa752c0, 0xc000338380}, 0xc0003208c0)
	/home/may/.go/pkg/mod/github.com/labstack/echo/[email protected]/echo.go:668 +0x323
net/http.serverHandler.ServeHTTP({0xc00031d3e0?}, {0xa752c0?, 0xc000338380?}, 0x6?)
	/usr/lib/go/src/net/http/server.go:3301 +0x8e
net/http.(*conn).serve(0xc0001575f0, {0xa76400, 0xc0001bd1d0})
	/usr/lib/go/src/net/http/server.go:2102 +0x625
created by net/http.(*Server).Serve in goroutine 1
	/usr/lib/go/src/net/http/server.go:3454 +0x485

goroutine 268 [runnable]:
net/http.(*connReader).startBackgroundRead.gowrap2()
	/usr/lib/go/src/net/http/server.go:686
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1
created by net/http.(*connReader).startBackgroundRead in goroutine 267
	/usr/lib/go/src/net/http/server.go:686 +0xb6
exit status 2
make: *** [Makefile:19: run] Error 1

I'm not entirely sure what information you'd need to help diagnose this. As far as I can tell the panicking stacktrace doesn't go through any of my code, and I'm not (intentionally) sharing the context between multiple goroutines anywhere.

[aldas]

does your code have any of their own goroutines? maybe event get reference to http.ResponseWriter from contexts passed to them somehow? What does that LiveRelationsConfig.Get do?

[edwloef]

I don't have any of my own goroutines, but I did notice yesterday that a custom middleware was moving a reference to the context to the main thread. I wasn't actually using it, but I've fixed that in hopes that that was the issue. I've found it pretty hard to trigger this specific crash though, it's only happened twice seemingly randomly (peak race condition fun)

[aldas]

It is very hard to tell anything specific - just search all the places you are calling c.Response() and doing something with it and asses if it could escape the previous request goroutine and/or lifetime of the request.