thanos-io/thanos · error

building gRPC client

Error message

building gRPC client

What it means

grpcClientConfig.dialOptions builds client-side gRPC dial options via extgrpc.StoreClientGRPCOpts, which assembles TLS credentials from the configured cert/key/CA files. When credential loading fails (unreadable files, bad PEM, invalid key pair), the error is wrapped as 'building gRPC client' and Thanos client setup aborts at startup.

Solutions

  1. Verify each TLS file path exists and is readable by the Thanos process; check k8s secret mounts and permissions.
  2. Validate that cert and key match: 'openssl x509 -noout -modulus -in cert.pem' vs 'openssl rsa -noout -modulus -in key.pem'.
  3. Confirm PEM contents are valid: 'openssl x509 -in cert.pem -text' and 'openssl verify -CAfile ca.pem cert.pem'.
  4. Migrate deprecated --grpc-client-tls-* flags to the YAML grpc client configuration (deprecated after v0.43.0), then restart.

Example fix

// before
--grpc-client-tls-cert=/etc/thanos/client.crt --grpc-client-tls-key=/etc/thanos/client.key --grpc-client-tls-ca=/etc/thanos/ca.crt
// (fails: /etc/thanos/ca.crt missing)
// after
--grpc-client-tls-cert=/etc/thanos/tls/client.crt --grpc-client-tls-key=/etc/thanos/tls/client.key --grpc-client-tls-ca=/etc/thanos/tls/ca.crt
Defensive patterns

Strategy: validation

Validate before calling

// Validate TLS material before dialing
func validateTLS(cert, key, ca string) error {
    for _, p := range []string{cert, key, ca} {
        if p == "" { continue }
        f, err := os.Open(p)
        if err != nil { return fmt.Errorf("cannot open %s: %w", p, err) }
        f.Close()
    }
    if cert != "" && key != "" {
        if _, err := tls.LoadX509KeyPair(cert, key); err != nil {
            return fmt.Errorf("invalid key pair: %w", err)
        }
    }
    return nil
}

Type guard

func tlsFilesReadable(cert, key, ca string) bool {
    for _, p := range []string{cert, key, ca} {
        if p != "" {
            if _, err := os.Stat(p); err != nil { return false }
        }
    }
    return true
}

Try / catch

dialOpts, err := gc.dialOptions(logger, reg, tracer)
if err != nil {
    return nil, errors.Wrap(err, "building gRPC client") // log full inner cause: file path + reason
}

Prevention

When it happens

Trigger: StoreClientGRPCOpts returns an error — cert, key, or CA file passed via --grpc-client-tls-cert/--grpc-client-tls-key/--grpc-client-tls-ca does not exist, is unreadable, or contains invalid PEM; or cert+key do not form a valid tls.X509KeyPair.

Common situations: Typo in TLS file paths, files mounted but with wrong permissions in Kubernetes secrets, expired/mismatched cert/key pairs, or using deprecated TLS flags after the v0.43.0 switch to YAML-based grpc client config without migrating.

Understand the failure class

Background: "failed to read file", EACCES, ENOENT and "could not read <path>" errors: when a program can't read a file from disk — this error's family across 49 libraries.

Related errors

AI-assisted analysis of thanos-io/thanos@35b8b99117 (2026-09-07). Data as JSON: /api/errors/56b2f2d806cac4cb. Report an issue: GitHub.

Appendix: source

Thrown at cmd/thanos/config.go:104

func (gc *grpcClientConfig) registerFlag(cmd extkingpin.FlagClause) *grpcClientConfig {
	cmd.Flag("grpc-client-tls-secure", "Deprecated after v0.43.0: Use TLS when talking to the gRPC server").Default("false").BoolVar(&gc.secure)
	cmd.Flag("grpc-client-tls-skip-verify", "Deprecated after v0.43.0: Disable TLS certificate verification i.e self signed, signed by fake CA").Default("false").BoolVar(&gc.skipVerify)
	cmd.Flag("grpc-client-tls-cert", "Deprecated after v0.43.0: TLS Certificates to use to identify this client to the server").Default("").StringVar(&gc.cert)
	cmd.Flag("grpc-client-tls-key", "Deprecated after v0.43.0: TLS Key for the client's certificate").Default("").StringVar(&gc.key)
	cmd.Flag("grpc-client-tls-ca", "Deprecated after v0.43.0: TLS CA Certificates to use to verify gRPC servers").Default("").StringVar(&gc.caCert)
	cmd.Flag("grpc-client-server-name", "Deprecated after v0.43.0: Server name to verify the hostname on the returned gRPC certificates. See https://tools.ietf.org/html/rfc4366#section-3.1").Default("").StringVar(&gc.serverName)
	compressionOptions := strings.Join([]string{snappy.Name, compressionNone}, ", ")
	cmd.Flag("grpc-compression", "Deprecated after v0.43.0: Compression algorithm to use for gRPC requests to other clients. Must be one of: "+compressionOptions).Default(compressionNone).EnumVar(&gc.compression, snappy.Name, compressionNone)
	cmd.Flag("grpc-client-tls-min-version",
		"Deprecated after v0.43.0: TLS supported minimum version for gRPC client. If no version is specified, it'll default to 1.3. Allowed values: [\"1.0\", \"1.1\", \"1.2\", \"1.3\"]").
		Default("1.3").EnumVar(&gc.minTLSVersion, tls.AllowedTLSVersions...)
	return gc
}

func (gc *grpcClientConfig) dialOptions(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer) ([]grpc.DialOption, error) {
	dialOpts, err := extgrpc.StoreClientGRPCOpts(logger, reg, tracer)
	if err != nil {
		return nil, errors.Wrapf(err, "building gRPC client")
	}
	return dialOpts, nil
}

type httpConfig struct {
	bindAddress string
	tlsConfig   string
	gracePeriod model.Duration
}

func (hc *httpConfig) registerFlag(cmd extkingpin.FlagClause) *httpConfig {
	cmd.Flag("http-address",
		"Listen host:port for HTTP endpoints.").
		Default("0.0.0.0:10902").StringVar(&hc.bindAddress)
	cmd.Flag("http-grace-period",
		"Time to wait after an interrupt received for HTTP Server.").
		Default("2m").SetValue(&hc.gracePeriod)
	cmd.Flag(

View on GitHub (pinned to 35b8b99117)