Verify for self signed ca

Fixes https://github.com/XTLS/Xray-core/issues/5506

app/dns/nameserver_udp.go

@@ -160,7 +160,7 @@ func (s *ClassicNameServer) getCacheController() *CacheController {
 }
 
 // sendQuery implements CachedNameserver.
-func (s *ClassicNameServer) sendQuery(ctx context.Context, _ chan<- error, fqdn string, option dns_feature.IPOption) {
+func (s *ClassicNameServer) sendQuery(ctx context.Context, noResponseErrCh chan<- error, fqdn string, option dns_feature.IPOption) {
 	errors.LogInfo(ctx, s.Name(), " querying DNS for: ", fqdn)
 
 	reqs := buildReqMsgs(fqdn, option, s.newReqID, genEDNS0Options(s.clientIP, 0))
@@ -171,7 +171,14 @@ func (s *ClassicNameServer) sendQuery(ctx context.Context, _ chan<- error, fqdn
 			ctx:        ctx,
 		}
 		s.addPendingRequest(udpReq)
-		b, _ := dns.PackMessage(req.msg)
+		b, err := dns.PackMessage(req.msg)
+		if err != nil {
+			errors.LogErrorInner(ctx, err, "failed to pack dns query")
+			if noResponseErrCh != nil {
+				noResponseErrCh <- err
+			}
+			return
+		}
 		copyDest := net.UDPDestination(s.address.Address, s.address.Port)
 		b.UDP = &copyDest
 		s.udpServer.Dispatch(toDnsContext(ctx, s.address.String()), *s.address, b)

transport/internet/tls/config.go

@@ -290,8 +290,10 @@ func (r *RandCarrier) verifyPeerCert(rawCerts [][]byte, verifiedChains [][]*x509
 	// directly return success if pinned cert is leaf
 	// or add the CA to RootCAs if pinned cert is CA(and can be used in VerifyPeerCertInNames for Self signed CA)
 	RootCAs := r.RootCAs
+	var verifyResult verifyResult
+	var verifiedCert *x509.Certificate
 	if r.PinnedPeerCertSha256 != nil {
-		verifyResult, verifiedCert := verifyChain(certs, r.PinnedPeerCertSha256)
+		verifyResult, verifiedCert = verifyChain(certs, r.PinnedPeerCertSha256)
 		switch verifyResult {
 		case certNotFound:
 			return errors.New("peer cert is unrecognized")
@@ -305,27 +307,39 @@ func (r *RandCarrier) verifyPeerCert(rawCerts [][]byte, verifiedChains [][]*x509
 		}
 	}
 
-	if r.VerifyPeerCertInNames != nil {
-		if len(r.VerifyPeerCertInNames) > 0 {
-			opts := x509.VerifyOptions{
-				Roots:         RootCAs,
-				CurrentTime:   time.Now(),
-				Intermediates: x509.NewCertPool(),
-			}
-			for _, cert := range certs[1:] {
-				opts.Intermediates.AddCert(cert)
-			}
-			for _, opts.DNSName = range r.VerifyPeerCertInNames {
-				if _, err := certs[0].Verify(opts); err == nil {
-					return nil
-				}
+	if len(r.VerifyPeerCertInNames) > 0 {
+		opts := x509.VerifyOptions{
+			Roots:         RootCAs,
+			CurrentTime:   time.Now(),
+			Intermediates: x509.NewCertPool(),
+		}
+		for _, cert := range certs[1:] {
+			opts.Intermediates.AddCert(cert)
+		}
+		for _, opts.DNSName = range r.VerifyPeerCertInNames {
+			if _, err := certs[0].Verify(opts); err == nil {
+				return nil
 			}
 		}
+	} else if len(verifiedChains) == 0 && verifyResult == foundCA { // if found ca and verifiedChains is empty, we need to verify here
+		opts := x509.VerifyOptions{
+			Roots:         RootCAs,
+			CurrentTime:   time.Now(),
+			Intermediates: x509.NewCertPool(),
+			DNSName:       r.Config.ServerName,
+		}
+		for _, cert := range certs[1:] {
+			opts.Intermediates.AddCert(cert)
+		}
+		if _, err := certs[0].Verify(opts); err == nil {
+			return nil
+		}
 	}
 	return nil
 }
 
 type RandCarrier struct {
+	Config                *tls.Config
 	RootCAs               *x509.CertPool
 	VerifyPeerCertInNames []string
 	PinnedPeerCertSha256  [][]byte
@@ -366,6 +380,7 @@ func (c *Config) GetTLSConfig(opts ...Option) *tls.Config {
 		SessionTicketsDisabled: !c.EnableSessionResumption,
 		VerifyPeerCertificate:  randCarrier.verifyPeerCert,
 	}
+	randCarrier.Config = config
 	if len(c.VerifyPeerCertInNames) > 0 {
 		config.InsecureSkipVerify = true
 	} else {