Recently, when I access the inspector, the session list is missing. F12 shows a HTTP 500 when accessing /openvidu/api/sessions?webRtcStats=true. After 10-20 attempts, the session list is eventually populated. I tried restarting the stack (2 master nodes, 2 media nodes) which initially made the problem disappear. However, it returned a day later.
Checking on the master node, docker logs of the openvidu/replication-manager show:
2025-01-30 22:33:31.848 INFO 8 --- [or-http-epoll-4] i.o.r.manager.filter.RequestFilter : GET /openvidu/api/sessions - Query params: {webRtcStats=[true]}
2025-01-30 22:33:31.949 ERROR 8 --- [or-http-epoll-2] a.w.r.e.AbstractErrorWebExceptionHandler : [d598d486-107652] 500 Server Error for HTTP GET "/openvidu/api/sessions?webRtcStats=true"
java.lang.IllegalStateException: Not a JSON Object: null
at com.google.gson.JsonElement.getAsJsonObject(JsonElement.java:91) ~[gson-2.9.1.jar!/:na]
Suppressed: reactor.core.publisher.FluxOnAssembly$OnAssemblyException:
Error has been observed at the following site(s):
*__checkpoint ⇢ io.openvidu.replication.manager.filter.PreFlightCorsFilter [DefaultWebFilterChain]
*__checkpoint ⇢ org.springframework.cloud.gateway.filter.WeightCalculatorWebFilter [DefaultWebFilterChain]
*__checkpoint ⇢ HTTP GET "/openvidu/api/sessions?webRtcStats=true" [ExceptionHandlingWebHandler]
Original Stack Trace:
at com.google.gson.JsonElement.getAsJsonObject(JsonElement.java:91) ~[gson-2.9.1.jar!/:na]
at io.openvidu.replication.manager.filter.RequestFilter.combineOvsResponses(RequestFilter.java:368) ~[classes!/:2.29.0]
at reactor.core.publisher.MonoReduce$ReduceSubscriber.onNext(MonoReduce.java:101) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxFlatMap$FlatMapMain.tryEmit(FluxFlatMap.java:543) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxFlatMap$FlatMapInner.onNext(FluxFlatMap.java:984) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.Operators$ScalarSubscription.request(Operators.java:2363) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.Operators$MultiSubscriptionSubscriber.set(Operators.java:2159) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onSubscribe(FluxOnErrorResume.java:74) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.MonoJust.subscribe(MonoJust.java:55) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.Mono.subscribe(Mono.java:4490) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onError(FluxOnErrorResume.java:103) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.MonoFlatMap$FlatMapMain.secondError(MonoFlatMap.java:192) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.MonoFlatMap$FlatMapInner.onError(MonoFlatMap.java:259) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxMap$MapSubscriber.onError(FluxMap.java:134) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.Operators$MonoSubscriber.onError(Operators.java:1849) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onError(FluxOnErrorResume.java:106) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.MonoFlatMap$FlatMapMain.onNext(MonoFlatMap.java:142) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxMap$MapSubscriber.onNext(FluxMap.java:122) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxOnErrorReturn$ReturnSubscriber.onError(FluxOnErrorReturn.java:199) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.Operators$MonoSubscriber.onError(Operators.java:1849) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.onError(FluxMapFuseable.java:142) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.onError(FluxContextWrite.java:121) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxMapFuseable$MapFuseableConditionalSubscriber.onError(FluxMapFuseable.java:340) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxFilterFuseable$FilterFuseableConditionalSubscriber.onError(FluxFilterFuseable.java:382) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.MonoCollect$CollectSubscriber.onError(MonoCollect.java:145) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxMap$MapSubscriber.onError(FluxMap.java:134) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxPeek$PeekSubscriber.onError(FluxPeek.java:222) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxMap$MapSubscriber.onError(FluxMap.java:134) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.Operators.error(Operators.java:198) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.netty.channel.FluxReceive.startReceiver(FluxReceive.java:187) ~[reactor-netty-core-1.0.25.jar!/:1.0.25]
at reactor.netty.channel.FluxReceive.subscribe(FluxReceive.java:148) ~[reactor-netty-core-1.0.25.jar!/:1.0.25]
at reactor.core.publisher.InternalFluxOperator.subscribe(InternalFluxOperator.java:62) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.netty.ByteBufFlux.subscribe(ByteBufFlux.java:340) ~[reactor-netty-core-1.0.25.jar!/:1.0.25]
at reactor.core.publisher.Mono.subscribe(Mono.java:4490) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onError(FluxOnErrorResume.java:103) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.onError(FluxMapFuseable.java:142) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.onError(FluxContextWrite.java:121) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxMapFuseable$MapFuseableConditionalSubscriber.onError(FluxMapFuseable.java:340) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxFilterFuseable$FilterFuseableConditionalSubscriber.onError(FluxFilterFuseable.java:382) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.MonoCollect$CollectSubscriber.onError(MonoCollect.java:145) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.MonoCollect$CollectSubscriber.onNext(MonoCollect.java:124) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxMap$MapSubscriber.onNext(FluxMap.java:122) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxPeek$PeekSubscriber.onNext(FluxPeek.java:200) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.core.publisher.FluxMap$MapSubscriber.onNext(FluxMap.java:122) ~[reactor-core-3.4.25.jar!/:3.4.25]
at reactor.netty.channel.FluxReceive.onInboundNext(FluxReceive.java:377) ~[reactor-netty-core-1.0.25.jar!/:1.0.25]
at reactor.netty.channel.ChannelOperations.onInboundNext(ChannelOperations.java:404) ~[reactor-netty-core-1.0.25.jar!/:1.0.25]
at reactor.netty.http.client.HttpClientOperations.onInboundNext(HttpClientOperations.java:724) ~[reactor-netty-http-1.0.25.jar!/:1.0.25]
at reactor.netty.channel.ChannelOperationsHandler.channelRead(ChannelOperationsHandler.java:113) ~[reactor-netty-core-1.0.25.jar!/:1.0.25]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:333) ~[netty-codec-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:454) ~[netty-codec-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:800) ~[netty-transport-classes-epoll-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:499) ~[netty-transport-classes-epoll-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[netty-transport-classes-epoll-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[netty-common-4.1.85.Final.jar!/:4.1.85.Final]
at java.base/java.lang.Thread.run(Unknown Source) ~[na:na]
2025-01-30 22:33:31.950 ERROR 8 --- [or-http-epoll-2] r.n.channel.ChannelOperationsHandler : [a7feb1e6-1, L:/172.24.1.112:58906 ! R:172.24.0.236/172.24.0.236:5443] Error was received while reading the incoming data. The connection will be closed.
io.netty.util.IllegalReferenceCountException: refCnt: 0, decrement: 1
at io.netty.util.internal.ReferenceCountUpdater.toLiveRealRefCnt(ReferenceCountUpdater.java:83) ~[netty-common-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.util.internal.ReferenceCountUpdater.release(ReferenceCountUpdater.java:147) ~[netty-common-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.buffer.AbstractReferenceCountedByteBuf.release(AbstractReferenceCountedByteBuf.java:101) ~[netty-buffer-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.handler.codec.http.DefaultHttpContent.release(DefaultHttpContent.java:92) ~[netty-codec-http-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.util.ReferenceCountUtil.release(ReferenceCountUtil.java:90) ~[netty-common-4.1.85.Final.jar!/:4.1.85.Final]
at reactor.netty.channel.FluxReceive.onInboundNext(FluxReceive.java:380) ~[reactor-netty-core-1.0.25.jar!/:1.0.25]
at reactor.netty.channel.ChannelOperations.onInboundNext(ChannelOperations.java:404) ~[reactor-netty-core-1.0.25.jar!/:1.0.25]
at reactor.netty.http.client.HttpClientOperations.onInboundNext(HttpClientOperations.java:724) ~[reactor-netty-http-1.0.25.jar!/:1.0.25]
at reactor.netty.channel.ChannelOperationsHandler.channelRead(ChannelOperationsHandler.java:113) ~[reactor-netty-core-1.0.25.jar!/:1.0.25]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:333) ~[netty-codec-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:454) ~[netty-codec-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:800) ~[netty-transport-classes-epoll-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:499) ~[netty-transport-classes-epoll-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[netty-transport-classes-epoll-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.85.Final.jar!/:4.1.85.Final]
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[netty-common-4.1.85.Final.jar!/:4.1.85.Final]
at java.base/java.lang.Thread.run(Unknown Source) ~[na:na]
2025-01-30 22:33:31.950 WARN 8 --- [or-http-epoll-2] reactor.netty.channel.FluxReceive : [a7feb1e6-1, L:/172.24.1.112:58906 ! R:172.24.0.236/172.24.0.236:5443] An exception has been observed post termination, use DEBUG level to see the full stack: io.netty.util.IllegalReferenceCountException: refCnt: 0, decrement: 1