enabling ssl causes grpc issues for zeebe gateway

Hello, I deployed camunda 8.1.6 in a centos vm on standalone mode, I deployed the gateway at port 26500 (without ssl), for me its mandatory to expose gateway, tasklist and operate in https, so I used nginx because I find it hard to do it based on the configuration, so I created some reverse proxies, they works fine for opearte and tasklist, but for gateway it gives some errors, I also created a spring boot application (api-bpm) that will comminucate with camunda, here is the main class :

@SpringBootApplication
@EnableFeignClients
@EnableZeebeClient
@ZeebeDeployment(resources = "classpath*:/bpmn/*.bpmn") //workflows are in resource folder
public class BpmApiApplication {

  public static void main(String[] args) {
    SpringApplication.run(BpmApiApplication.class, args);
  }
}

when I run the project I got thoses issues :

Error starting ApplicationContext. To display the conditions report re-run your application with 'debug' enabled.
2023-05-31 11:17:27.109 ERROR [axa-ma-inner-api-bpm,,] 35044 --- [  restartedMain] o.s.boot.SpringApplication               : Application run failed

org.springframework.context.ApplicationContextException: Failed to start bean 'zeebeClientLifecycle'; nested exception is io.camunda.zeebe.client.api.command.ClientStatusException: http2 exception
	at org.springframework.context.support.DefaultLifecycleProcessor.doStart(DefaultLifecycleProcessor.java:181) ~[spring-context-5.3.15.jar:5.3.15]
	at org.springframework.context.support.DefaultLifecycleProcessor.access$200(DefaultLifecycleProcessor.java:54) ~[spring-context-5.3.15.jar:5.3.15]
	at org.springframework.context.support.DefaultLifecycleProcessor$LifecycleGroup.start(DefaultLifecycleProcessor.java:356) ~[spring-context-5.3.15.jar:5.3.15]
	at java.base/java.lang.Iterable.forEach(Iterable.java:75) ~[na:na]
	at org.springframework.context.support.DefaultLifecycleProcessor.startBeans(DefaultLifecycleProcessor.java:155) ~[spring-context-5.3.15.jar:5.3.15]
	at org.springframework.context.support.DefaultLifecycleProcessor.onRefresh(DefaultLifecycleProcessor.java:123) ~[spring-context-5.3.15.jar:5.3.15]
	at org.springframework.context.support.AbstractApplicationContext.finishRefresh(AbstractApplicationContext.java:935) ~[spring-context-5.3.15.jar:5.3.15]
	at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:586) ~[spring-context-5.3.15.jar:5.3.15]
	at org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext.refresh(ServletWebServerApplicationContext.java:145) ~[spring-boot-2.6.7.jar:2.6.7]
	at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:740) ~[spring-boot-2.6.7.jar:2.6.7]
	at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:415) ~[spring-boot-2.6.7.jar:2.6.7]
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:303) ~[spring-boot-2.6.7.jar:2.6.7]
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1312) ~[spring-boot-2.6.7.jar:2.6.7]
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1301) ~[spring-boot-2.6.7.jar:2.6.7]
	at ma.axa.inner.bpm.AxaInnerBpmApiApplication.main(AxaInnerBpmApiApplication.java:16) ~[classes/:na]
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:na]
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:na]
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:na]
	at java.base/java.lang.reflect.Method.invoke(Method.java:566) ~[na:na]
	at org.springframework.boot.devtools.restart.RestartLauncher.run(RestartLauncher.java:49) ~[spring-boot-devtools-2.6.7.jar:2.6.7]
Caused by: io.camunda.zeebe.client.api.command.ClientStatusException: http2 exception
	at io.camunda.zeebe.client.impl.ZeebeClientFutureImpl.transformExecutionException(ZeebeClientFutureImpl.java:93) ~[zeebe-client-java-8.1.6.jar:8.1.6]
	at io.camunda.zeebe.client.impl.ZeebeClientFutureImpl.join(ZeebeClientFutureImpl.java:50) ~[zeebe-client-java-8.1.6.jar:8.1.6]
	at io.camunda.zeebe.spring.client.annotation.processor.ZeebeDeploymentAnnotationProcessor.lambda$start$7(ZeebeDeploymentAnnotationProcessor.java:119) ~[spring-zeebe-8.1.6.jar:8.1.6]
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1541) ~[na:na]
	at io.camunda.zeebe.spring.client.annotation.processor.ZeebeDeploymentAnnotationProcessor.start(ZeebeDeploymentAnnotationProcessor.java:100) ~[spring-zeebe-8.1.6.jar:8.1.6]
	at io.camunda.zeebe.spring.client.annotation.processor.ZeebeAnnotationProcessorRegistry.lambda$startAll$0(ZeebeAnnotationProcessorRegistry.java:38) ~[spring-zeebe-8.1.6.jar:8.1.6]
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1541) ~[na:na]
	at io.camunda.zeebe.spring.client.annotation.processor.ZeebeAnnotationProcessorRegistry.startAll(ZeebeAnnotationProcessorRegistry.java:38) ~[spring-zeebe-8.1.6.jar:8.1.6]
	at io.camunda.zeebe.spring.client.lifecycle.ZeebeClientLifecycle.start(ZeebeClientLifecycle.java:49) ~[spring-zeebe-8.1.6.jar:8.1.6]
	at org.springframework.context.support.DefaultLifecycleProcessor.doStart(DefaultLifecycleProcessor.java:178) ~[spring-context-5.3.15.jar:5.3.15]
	... 19 common frames omitted
Caused by: java.util.concurrent.ExecutionException: io.grpc.StatusRuntimeException: INTERNAL: http2 exception
	at java.base/java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:395) ~[na:na]
	at java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1999) ~[na:na]
	at io.camunda.zeebe.client.impl.ZeebeClientFutureImpl.join(ZeebeClientFutureImpl.java:48) ~[zeebe-client-java-8.1.6.jar:8.1.6]
	... 27 common frames omitted
Caused by: io.grpc.StatusRuntimeException: INTERNAL: http2 exception
	at io.grpc.Status.asRuntimeException(Status.java:535) ~[grpc-api-1.49.1.jar:1.49.1]
	at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:487) ~[grpc-stub-1.49.1.jar:1.49.1]
	at io.grpc.internal.DelayedClientCall$DelayedListener$3.run(DelayedClientCall.java:470) ~[grpc-core-1.49.1.jar:1.49.1]
	at io.grpc.internal.DelayedClientCall$DelayedListener.delayOrExecute(DelayedClientCall.java:434) ~[grpc-core-1.49.1.jar:1.49.1]
	at io.grpc.internal.DelayedClientCall$DelayedListener.onClose(DelayedClientCall.java:467) ~[grpc-core-1.49.1.jar:1.49.1]
	at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:563) ~[grpc-core-1.49.1.jar:1.49.1]
	at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:70) ~[grpc-core-1.49.1.jar:1.49.1]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:744) ~[grpc-core-1.49.1.jar:1.49.1]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:723) ~[grpc-core-1.49.1.jar:1.49.1]
	at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) ~[grpc-core-1.49.1.jar:1.49.1]
	at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133) ~[grpc-core-1.49.1.jar:1.49.1]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[na:na]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[na:na]
	at java.base/java.lang.Thread.run(Thread.java:834) ~[na:na]
Caused by: io.netty.handler.codec.http2.Http2Exception: First received frame was not SETTINGS. Hex dump for first 5 bytes: 485454502f
	at io.netty.handler.codec.http2.Http2Exception.connectionError(Http2Exception.java:108) ~[netty-codec-http2-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.handler.codec.http2.Http2ConnectionHandler$PrefaceDecoder.verifyFirstFrameIsSettings(Http2ConnectionHandler.java:338) ~[netty-codec-http2-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.handler.codec.http2.Http2ConnectionHandler$PrefaceDecoder.decode(Http2ConnectionHandler.java:239) ~[netty-codec-http2-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.handler.codec.http2.Http2ConnectionHandler.decode(Http2ConnectionHandler.java:438) ~[netty-codec-http2-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510) ~[netty-codec-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449) ~[netty-codec-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) ~[netty-codec-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[netty-transport-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[netty-transport-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[netty-transport-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[netty-transport-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[netty-transport-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) ~[netty-transport-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:658) ~[netty-transport-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:584) ~[netty-transport-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) ~[netty-transport-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) ~[netty-common-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.73.Final.jar:4.1.73.Final]
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[netty-common-4.1.73.Final.jar:4.1.73.Final]
	... 1 common frames omitted


Process finished with exit code 130

here is my nginx configuration :

upstream dev {
    server localhost:26500;
}

server {
    listen 448 ssl http2;
    ssl_certificate /etc/nginx/conf.d/certs/server.crt;
    ssl_certificate_key /etc/nginx/conf.d/certs/server.key;
    # gRPC specific settings
    grpc_ssl_certificate /etc/nginx/conf.d/certs/server.crt;  # Replace with your SSL certificate file path
    grpc_ssl_certificate_key /etc/nginx/conf.d/certs/server.key;  # Replace with your SSL private key file path
    grpc_ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;  # Enable specific SSL protocols as needed
    grpc_ssl_ciphers HIGH:!aNULL:!MD5;  # Adjust SSL cipher suites as needed

    location / {
       grpc_pass grpc://127.0.0.1:26500;
       grpc_set_header Host $host;
       grpc_set_header X-Real-IP $remote_addr;
       grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       grpc_set_header X-Forwarded-Proto $scheme;
    }
}

nginx conf is basic, just forward the request (http2 & ssl) to the gateway.

this is the application.yml of api-bpm (I deplicated the conf of security in broker and gateway based on env var used in linux, not sure is correct), for the certs the are the same used by nginx :

server:
  port: ${BPM_PORT:8087}

### Zeebee properties ###
zeebe:
  client:
    broker:
      gateway:
        security:
          plaintext: true
          ## Enables TLS authentication between clients and the gateway
          enabled: true
          ## Sets the path to the certificate chain file
          certificateChainPath: C:\Users\comp\server.crt
          ## Sets the path to the private key file location
          privateKeyPath: C:\Users\comp\server.key
      gatewayAddress: 192.168.4.194:448
    security:
      plaintext: true
## Enables TLS authentication between clients and the gateway
      enabled: true
## Sets the path to the certificate chain file
      certificateChainPath: C:\Users\comp\server.crt
## Sets the path to the private key file location
      privateKeyPath: C:\Users\comp\server.key

### DOC prop ###
springdoc:
  paths-to-match: /**
  api-docs:
    path: "/docs/api-docs"
    enabled: true
  swagger-ui:
    path: "/docs/swagger-ui.html"
    url: "${axa.context}${springdoc.api-docs.path}"
    configUrl: "${axa.context}/docs/api-docs/swagger-config"
    disable-swagger-default-url: true
    operationsSorter: alpha
    tagsSorter: alpha
    
### SPRING ###
spring:
  application:
    name: "@project.name@"
  profiles:
    default: local
    group:
      local: local,api-docs
      dev: dev,api-docs
      uat: uat,api-docs
      
  security:
    oauth2:
      resourceserver:
        jwt:
          jwk-set-uri: ${AUTH_ADFS_URL_KEYS:https://srv-dev.local/adfs/discovery/keys}

### LOG LEVEL ###
logging:
  level:
    reactor:
      ipc:
        netty:
          channel:
            CloseableContextHandler=off:
    root: INFO
    ma.axa.inner: DEBUG
    org.springframework.web: DEBUG
    org.hibernate: ERROR
    org.springframework.security: INFO

    
### INFO ENV ###
info:
  application:
    name: ${spring.application.name}
    version: "@project.version@"
    
### API MANAGEMENT ###
management:
  endpoint:
    prometheus:
      enabled: true
    health:
      enabled: true
      show-details: always
  info:
    env:
      enabled: true
    java:
      enabled: true
  endpoints:
    web:
      exposure:
        include: health,info,prometheus,metrics
      base-path: /management
  metrics:
    tags:
      application: ${spring.application.name}

---

spring:
  application:
    name: "@project.name@"
  cloud:
    config:
      enabled: false

feign:
  client:
    config:
      default:
        connectTimeout: 15000
        readTimeout: 20000
        loggerLevel: basic

### CUSTOM prop ###

  bpmn:
    username: ${BPM_USERNAME:demo}
    password: ${BPM_PASSWORD:demo}
    taskListUrl: https://192.168.4.194:446
    operateUrl: https://192.168.4.194:444
  security:
    enabled: true
    allowed-origin: "*"
  docs:
    version: "@project.version@"
  context: "/bpm"

Actually for the same configuration if I disable the ssl via nginx it works fine :


upstream dev {
    server localhost:26500;
}

server {
    listen 448 http2;
    ssl_certificate /etc/nginx/conf.d/certs/server.crt;
    ssl_certificate_key /etc/nginx/conf.d/certs/server.key;
    # gRPC specific settings
    grpc_ssl_certificate /etc/nginx/conf.d/certs/server.crt;  # Replace with your SSL certificate file path
    grpc_ssl_certificate_key /etc/nginx/conf.d/certs/server.key;  # Replace with your SSL private key file path
    grpc_ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;  # Enable specific SSL protocols as needed
    grpc_ssl_ciphers HIGH:!aNULL:!MD5;  # Adjust SSL cipher suites as needed

    location / {
       grpc_pass grpc://127.0.0.1:26500;
       grpc_set_header Host $host;
       grpc_set_header X-Real-IP $remote_addr;
       grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       grpc_set_header X-Forwarded-Proto $scheme;
    }
}

If I recall other threads on the forum correctly, you cannot have both plaintext:true and use SSL.
Iā€™d suggest taking the plaintext: true lines out of your client config and trying again.

1 Like

I did but I still got the same issues

1 Like