Verify for self signed ca

Fixes https://github.com/XTLS/Xray-core/issues/5506

app/dns/nameserver_udp.go

@@ -160,7 +160,7 @@ func (s *ClassicNameServer) getCacheController() *CacheController {
 }
 
 // sendQuery implements CachedNameserver.
-func (s *ClassicNameServer) sendQuery(ctx context.Context, _ chan<- error, fqdn string, option dns_feature.IPOption) {
+func (s *ClassicNameServer) sendQuery(ctx context.Context, noResponseErrCh chan<- error, fqdn string, option dns_feature.IPOption) {
 	errors.LogInfo(ctx, s.Name(), " querying DNS for: ", fqdn)
 
 	reqs := buildReqMsgs(fqdn, option, s.newReqID, genEDNS0Options(s.clientIP, 0))
@@ -171,7 +171,14 @@ func (s *ClassicNameServer) sendQuery(ctx context.Context, _ chan<- error, fqdn
 			ctx:        ctx,
 		}
 		s.addPendingRequest(udpReq)
-		b, _ := dns.PackMessage(req.msg)
+		b, err := dns.PackMessage(req.msg)
+		if err != nil {
+			errors.LogErrorInner(ctx, err, "failed to pack dns query")
+			if noResponseErrCh != nil {
+				noResponseErrCh <- err
+			}
+			return
+		}
 		copyDest := net.UDPDestination(s.address.Address, s.address.Port)
 		b.UDP = &copyDest
 		s.udpServer.Dispatch(toDnsContext(ctx, s.address.String()), *s.address, b)

go.mod

@@ -25,7 +25,6 @@ require (
 	golang.org/x/net v0.48.0
 	golang.org/x/sync v0.19.0
 	golang.org/x/sys v0.39.0
-	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173
 	google.golang.org/grpc v1.78.0
 	google.golang.org/protobuf v1.36.11
@@ -51,6 +50,7 @@ require (
 	golang.org/x/text v0.32.0 // indirect
 	golang.org/x/time v0.12.0 // indirect
 	golang.org/x/tools v0.39.0 // indirect
+	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

proxy/wireguard/bind.go

@@ -124,26 +124,6 @@ type netBindClient struct {
 	ctx      context.Context
 	dialer   internet.Dialer
 	reserved []byte
-	
-	// Track all peer connections for unified reading
-	connMutex sync.RWMutex
-	conns     map[*netEndpoint]net.Conn
-	dataChan  chan *receivedData
-	closeChan chan struct{}
-	closeOnce sync.Once
-}
-
-const (
-	// Buffer size for dataChan - allows some buffering of received packets
-	// while dispatcher matches them with read requests
-	dataChannelBufferSize = 100
-)
-
-type receivedData struct {
-	data     []byte
-	n        int
-	endpoint *netEndpoint
-	err      error
 }
 
 func (bind *netBindClient) connectTo(endpoint *netEndpoint) error {
@@ -153,114 +133,34 @@ func (bind *netBindClient) connectTo(endpoint *netEndpoint) error {
 	}
 	endpoint.conn = c
 
-	// Initialize channels on first connection
-	bind.connMutex.Lock()
-	if bind.conns == nil {
-		bind.conns = make(map[*netEndpoint]net.Conn)
-		bind.dataChan = make(chan *receivedData, dataChannelBufferSize)
-		bind.closeChan = make(chan struct{})
-		
-		// Start unified reader dispatcher
-		go bind.unifiedReader()
-	}
-	bind.conns[endpoint] = c
-	bind.connMutex.Unlock()
-	
-	// Start a reader goroutine for this specific connection
-	go func(conn net.Conn, endpoint *netEndpoint) {
-		const maxPacketSize = 1500
+	go func(readQueue <-chan *netReadInfo, endpoint *netEndpoint) {
 		for {
-			select {
-			case <-bind.closeChan:
-				return
-			default:
-			}
-			
-			buf := make([]byte, maxPacketSize)
-			n, err := conn.Read(buf)
-			
-			// Send only the valid data portion to dispatcher
-			dataToSend := buf
-			if n > 0 && n < len(buf) {
-				dataToSend = buf[:n]
-			}
-			
-			// Send received data to dispatcher
-			select {
-			case bind.dataChan <- &receivedData{
-				data:     dataToSend,
-				n:        n,
-				endpoint: endpoint,
-				err:      err,
-			}:
-			case <-bind.closeChan:
+			v, ok := <-readQueue
+			if !ok {
 				return
 			}
-			
+			i, err := c.Read(v.buff)
+
+			if i > 3 {
+				v.buff[1] = 0
+				v.buff[2] = 0
+				v.buff[3] = 0
+			}
+
+			v.bytes = i
+			v.endpoint = endpoint
+			v.err = err
+			v.waiter.Done()
 			if err != nil {
-				bind.connMutex.Lock()
-				delete(bind.conns, endpoint)
 				endpoint.conn = nil
-				bind.connMutex.Unlock()
 				return
 			}
 		}
-	}(c, endpoint)
+	}(bind.readQueue, endpoint)
 
 	return nil
 }
 
-// unifiedReader dispatches received data to waiting read requests
-func (bind *netBindClient) unifiedReader() {
-	for {
-		select {
-		case data := <-bind.dataChan:
-			// Bounds check to prevent panic
-			if data.n > len(data.data) {
-				data.n = len(data.data)
-			}
-			
-			// Wait for a read request with timeout to prevent blocking forever
-			select {
-			case v := <-bind.readQueue:
-				// Copy data to request buffer
-				n := copy(v.buff, data.data[:data.n])
-				
-				// Clear reserved bytes if needed
-				if n > 3 {
-					v.buff[1] = 0
-					v.buff[2] = 0
-					v.buff[3] = 0
-				}
-				
-				v.bytes = n
-				v.endpoint = data.endpoint
-				v.err = data.err
-				v.waiter.Done()
-			case <-bind.closeChan:
-				return
-			}
-		case <-bind.closeChan:
-			return
-		}
-	}
-}
-
-// Close implements conn.Bind.Close for netBindClient
-func (bind *netBindClient) Close() error {
-	// Use sync.Once to prevent double-close panic
-	bind.closeOnce.Do(func() {
-		bind.connMutex.Lock()
-		if bind.closeChan != nil {
-			close(bind.closeChan)
-		}
-		bind.connMutex.Unlock()
-	})
-	
-	// Call parent Close
-	return bind.netBind.Close()
-}
-
 func (bind *netBindClient) Send(buff [][]byte, endpoint conn.Endpoint) error {
 	var err error
 

proxy/wireguard/client.go

@@ -114,12 +114,6 @@ func (h *Handler) processWireGuard(ctx context.Context, dialer internet.Dialer)
 	}
 
 	// bind := conn.NewStdNetBind() // TODO: conn.Bind wrapper for dialer
-	// Set workers to number of peers if not explicitly configured
-	// This allows concurrent packet reception from multiple peers
-	workers := int(h.conf.NumWorkers)
-	if workers <= 0 && len(h.conf.Peers) > 0 {
-		workers = len(h.conf.Peers)
-	}
 	h.bind = &netBindClient{
 		netBind: netBind{
 			dns: h.dns,
@@ -127,9 +121,9 @@ func (h *Handler) processWireGuard(ctx context.Context, dialer internet.Dialer)
 				IPv4Enable: h.hasIPv4,
 				IPv6Enable: h.hasIPv6,
 			},
-			workers: workers,
+			workers: int(h.conf.NumWorkers),
 		},
-		ctx:      core.ToBackgroundDetachedContext(ctx),
+		ctx:      ctx,
 		dialer:   dialer,
 		reserved: h.conf.Reserved,
 	}

transport/internet/tls/config.go

@@ -290,8 +290,10 @@ func (r *RandCarrier) verifyPeerCert(rawCerts [][]byte, verifiedChains [][]*x509
 	// directly return success if pinned cert is leaf
 	// or add the CA to RootCAs if pinned cert is CA(and can be used in VerifyPeerCertInNames for Self signed CA)
 	RootCAs := r.RootCAs
+	var verifyResult verifyResult
+	var verifiedCert *x509.Certificate
 	if r.PinnedPeerCertSha256 != nil {
-		verifyResult, verifiedCert := verifyChain(certs, r.PinnedPeerCertSha256)
+		verifyResult, verifiedCert = verifyChain(certs, r.PinnedPeerCertSha256)
 		switch verifyResult {
 		case certNotFound:
 			return errors.New("peer cert is unrecognized")
@@ -305,27 +307,39 @@ func (r *RandCarrier) verifyPeerCert(rawCerts [][]byte, verifiedChains [][]*x509
 		}
 	}
 
-	if r.VerifyPeerCertInNames != nil {
-		if len(r.VerifyPeerCertInNames) > 0 {
-			opts := x509.VerifyOptions{
-				Roots:         RootCAs,
-				CurrentTime:   time.Now(),
-				Intermediates: x509.NewCertPool(),
-			}
-			for _, cert := range certs[1:] {
-				opts.Intermediates.AddCert(cert)
-			}
-			for _, opts.DNSName = range r.VerifyPeerCertInNames {
-				if _, err := certs[0].Verify(opts); err == nil {
-					return nil
-				}
+	if len(r.VerifyPeerCertInNames) > 0 {
+		opts := x509.VerifyOptions{
+			Roots:         RootCAs,
+			CurrentTime:   time.Now(),
+			Intermediates: x509.NewCertPool(),
+		}
+		for _, cert := range certs[1:] {
+			opts.Intermediates.AddCert(cert)
+		}
+		for _, opts.DNSName = range r.VerifyPeerCertInNames {
+			if _, err := certs[0].Verify(opts); err == nil {
+				return nil
 			}
 		}
+	} else if len(verifiedChains) == 0 && verifyResult == foundCA { // if found ca and verifiedChains is empty, we need to verify here
+		opts := x509.VerifyOptions{
+			Roots:         RootCAs,
+			CurrentTime:   time.Now(),
+			Intermediates: x509.NewCertPool(),
+			DNSName:       r.Config.ServerName,
+		}
+		for _, cert := range certs[1:] {
+			opts.Intermediates.AddCert(cert)
+		}
+		if _, err := certs[0].Verify(opts); err == nil {
+			return nil
+		}
 	}
 	return nil
 }
 
 type RandCarrier struct {
+	Config                *tls.Config
 	RootCAs               *x509.CertPool
 	VerifyPeerCertInNames []string
 	PinnedPeerCertSha256  [][]byte
@@ -366,6 +380,7 @@ func (c *Config) GetTLSConfig(opts ...Option) *tls.Config {
 		SessionTicketsDisabled: !c.EnableSessionResumption,
 		VerifyPeerCertificate:  randCarrier.verifyPeerCert,
 	}
+	randCarrier.Config = config
 	if len(c.VerifyPeerCertInNames) > 0 {
 		config.InsecureSkipVerify = true
 	} else {