diff --git a/listeners.go b/listeners.go
index c0d018bb..01adc615 100644
--- a/listeners.go
+++ b/listeners.go
@@ -430,6 +430,7 @@ func JoinNetworkAddress(network, host, port string) string {
 // address instead.
 //
 // NOTE: This API is EXPERIMENTAL and may be changed or removed.
+// NOTE: user should close the returned listener twice, once to stop accepting new connections, the second time to free up the packet conn.
 func (na NetworkAddress) ListenQUIC(ctx context.Context, portOffset uint, config net.ListenConfig, tlsConf *tls.Config) (http3.QUICListener, error) {
 	lnKey := listenerKey("quic"+na.Network, na.JoinHostPort(portOffset))
 
@@ -626,6 +627,7 @@ func (fcql *fakeCloseQuicListener) Accept(_ context.Context) (*quic.Conn, error)
 func (fcql *fakeCloseQuicListener) Close() error {
 	if atomic.CompareAndSwapInt32(&fcql.closed, 0, 1) {
 		fcql.contextCancel()
+	} else if atomic.CompareAndSwapInt32(&fcql.closed, 1, 2) {
 		_, _ = listenerPool.Delete(fcql.sharedQuicListener.key)
 	}
 	return nil
diff --git a/modules/caddyhttp/app.go b/modules/caddyhttp/app.go
index 3e14ddb2..afa6cd0f 100644
--- a/modules/caddyhttp/app.go
+++ b/modules/caddyhttp/app.go
@@ -722,11 +722,29 @@ func (app *App) Stop() error {
 			return
 		}
 
+		// closing quic listeners won't affect accepted connections now
+		// so like stdlib, close listeners first, but keep the net.PacketConns open
+		for _, h3ln := range server.quicListeners {
+			if err := h3ln.Close(); err != nil {
+				app.logger.Error("http3 listener close",
+					zap.Error(err))
+			}
+		}
+
 		if err := server.h3server.Shutdown(ctx); err != nil {
 			app.logger.Error("HTTP/3 server shutdown",
 				zap.Error(err),
 				zap.Strings("addresses", server.Listen))
 		}
+
+		// close the underlying net.PacketConns now
+		// see the comment for ListenQUIC
+		for _, h3ln := range server.quicListeners {
+			if err := h3ln.Close(); err != nil {
+				app.logger.Error("http3 listener close socket",
+					zap.Error(err))
+			}
+		}
 	}
 	stopH2Listener := func(server *Server) {
 		defer finishedShutdown.Done()
diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go
index a2b29d65..069807b2 100644
--- a/modules/caddyhttp/server.go
+++ b/modules/caddyhttp/server.go
@@ -235,7 +235,8 @@ type Server struct {
 	primaryHandlerChain Handler
 	errorHandlerChain   Handler
 	listenerWrappers    []caddy.ListenerWrapper
-	listeners           []net.Listener
+	listeners           []net.Listener       // stdlib http.Server will close these
+	quicListeners       []http3.QUICListener // http3 now leave the quic.Listener management to us
 
 	tlsApp       *caddytls.TLS
 	events       *caddyevents.App
@@ -626,6 +627,8 @@ func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error
 		}
 	}
 
+	s.quicListeners = append(s.quicListeners, h3ln)
+
 	//nolint:errcheck
 	go s.h3server.ServeListener(h3ln)