Add 'Recovery' middleware (grpc-ecosystem#30)

Adding middleware that recovers from panicking server implementations
 instead producing a gRPC error

README.md

@@ -36,13 +36,15 @@ myServer := grpc.NewServer(
         grpc_prometheus.StreamServerInterceptor,
         grpc_zap.StreamServerInterceptor(zapLogger),
         grpc_auth.StreamServerInterceptor(myAuthFunction),
+        grpc_recovery.StreamServerInterceptor(),
     )),
     grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
         grpc_ctxtags.UnaryServerInterceptor(),
         grpc_opentracing.UnaryServerInterceptor(),
         grpc_prometheus.UnaryServerInterceptor,
         grpc_zap.UnaryServerInterceptor(zapLogger),
         grpc_auth.UnaryServerInterceptor(myAuthFunction),
+        grpc_recovery.UnaryServerInterceptor(),
     )),
 )
 ```
@@ -69,7 +71,8 @@ myServer := grpc.NewServer(
    * [`grpc_retry`](retry/) - a generic gRPC response code retry mechanism, client-side middleware
 
 #### Server
-   * [`grpc_validator`](validator/) - codegen inbound message validation from `.proto` options 
+   * [`grpc_validator`](validator/) - codegen inbound message validation from `.proto` options
+   * [`grpc_recovery`](recovery/) - turn panics into gRPC errors
 
 
 ## Status

recovery/DOC.md

@@ -0,0 +1,55 @@
+# grpc_recovery
+--
+    import "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
+
+`grpc_recovery` conversion of panics into gRPC errors
+
+
+### Server Side Recovery Middleware
+
+By default a panic will be converted into a gRPC error with `code.Internal`.
+
+Handling can be customised by providing an alternate recovery function.
+
+Please see examples for simple examples of use.
+
+## Usage
+
+#### func  StreamServerInterceptor
+
+```go
+func StreamServerInterceptor(opts ...Option) grpc.StreamServerInterceptor
+```
+StreamServerInterceptor returns a new streaming server interceptor for panic
+recovery.
+
+#### func  UnaryServerInterceptor
+
+```go
+func UnaryServerInterceptor(opts ...Option) grpc.UnaryServerInterceptor
+```
+UnaryServerInterceptor returns a new unary server interceptor for panic
+recovery.
+
+#### type Option
+
+```go
+type Option func(*options)
+```
+
+
+#### func  WithRecoveryHandler
+
+```go
+func WithRecoveryHandler(f RecoveryHandlerFunc) Option
+```
+WithRecoveryHandler customizes the function for recovering from a panic.
+
+#### type RecoveryHandlerFunc
+
+```go
+type RecoveryHandlerFunc func(p interface{}) (err error)
+```
+
+RecoveryHandlerFunc is a function that recovers from the panic `p` by returning
+an `error`.

recovery/README.md

@@ -0,0 +1 @@
+DOC.md

recovery/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2017 David Ackroyd. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+/*
+`grpc_recovery` conversion of panics into gRPC errors
+
+Server Side Recovery Middleware
+
+By default a panic will be converted into a gRPC error with `code.Internal`.
+
+Handling can be customised by providing an alternate recovery function.
+
+Please see examples for simple examples of use.
+*/
+package grpc_recovery

recovery/examples_test.go

@@ -0,0 +1,29 @@
+// Copyright 2017 David Ackroyd. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+package grpc_recovery_test
+
+import (
+	"github.com/grpc-ecosystem/go-grpc-middleware"
+	"github.com/grpc-ecosystem/go-grpc-middleware/recovery"
+	"google.golang.org/grpc"
+)
+
+// Initialization shows an initialization sequence with a custom recovery handler func.
+func Example_initialization(customFunc grpc_recovery.RecoveryHandlerFunc) *grpc.Server {
+	// Shared options for the logger, with a custom gRPC code to log level function.
+	opts := []grpc_recovery.Option{
+		grpc_recovery.WithRecoveryHandler(customFunc),
+	}
+	// Create a server. Recovery handlers should typically be last in the chain so that other middleware
+	// (e.g. logging) can operate on the recovered state instead of being directly affected by any panic
+	server := grpc.NewServer(
+		grpc_middleware.WithUnaryServerChain(
+			grpc_recovery.UnaryServerInterceptor(opts...),
+		),
+		grpc_middleware.WithStreamServerChain(
+			grpc_recovery.StreamServerInterceptor(opts...),
+		),
+	)
+	return server
+}

recovery/interceptors.go

@@ -0,0 +1,48 @@
+// Copyright 2017 David Ackroyd. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+package grpc_recovery
+
+import (
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+)
+
+// RecoveryHandlerFunc is a function that recovers from the panic `p` by returning an `error`.
+type RecoveryHandlerFunc func(p interface{}) (err error)
+
+// UnaryServerInterceptor returns a new unary server interceptor for panic recovery.
+func UnaryServerInterceptor(opts ...Option) grpc.UnaryServerInterceptor {
+	o := evaluateOptions(opts)
+	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (_ interface{}, err error) {
+		defer func() {
+			if r := recover(); r != nil {
+				err = recoverFrom(r, o.recoveryHandlerFunc)
+			}
+		}()
+
+		return handler(ctx, req)
+	}
+}
+
+// StreamServerInterceptor returns a new streaming server interceptor for panic recovery.
+func StreamServerInterceptor(opts ...Option) grpc.StreamServerInterceptor {
+	o := evaluateOptions(opts)
+	return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) (err error) {
+		defer func() {
+			if r := recover(); r != nil {
+				err = recoverFrom(r, o.recoveryHandlerFunc)
+			}
+		}()
+
+		return handler(srv, stream)
+	}
+}
+
+func recoverFrom(p interface{}, r RecoveryHandlerFunc) error {
+	if r == nil {
+		return grpc.Errorf(codes.Internal, "%s", p)
+	}
+	return r(p)
+}

recovery/interceptors_test.go

@@ -0,0 +1,143 @@
+// Copyright 2017 David Ackroyd. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+package grpc_recovery_test
+
+import (
+	"testing"
+
+	"github.com/grpc-ecosystem/go-grpc-middleware"
+	"github.com/grpc-ecosystem/go-grpc-middleware/recovery"
+	"github.com/grpc-ecosystem/go-grpc-middleware/testing"
+	pb_testproto "github.com/grpc-ecosystem/go-grpc-middleware/testing/testproto"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+)
+
+var (
+	goodPing  = &pb_testproto.PingRequest{Value: "something", SleepTimeMs: 9999}
+	panicPing = &pb_testproto.PingRequest{Value: "panic", SleepTimeMs: 9999}
+)
+
+type recoveryAssertService struct {
+	pb_testproto.TestServiceServer
+}
+
+func (s *recoveryAssertService) Ping(ctx context.Context, ping *pb_testproto.PingRequest) (*pb_testproto.PingResponse, error) {
+	if ping.Value == "panic" {
+		panic("very bad thing happened")
+	}
+	return s.TestServiceServer.Ping(ctx, ping)
+}
+
+func (s *recoveryAssertService) PingList(ping *pb_testproto.PingRequest, stream pb_testproto.TestService_PingListServer) error {
+	if ping.Value == "panic" {
+		panic("very bad thing happened")
+	}
+	return s.TestServiceServer.PingList(ping, stream)
+}
+
+func TestRecoverySuite(t *testing.T) {
+	s := &RecoverySuite{
+		InterceptorTestSuite: &grpc_testing.InterceptorTestSuite{
+			TestService: &recoveryAssertService{TestServiceServer: &grpc_testing.TestPingService{T: t}},
+			ServerOpts: []grpc.ServerOption{
+				grpc_middleware.WithStreamServerChain(
+					grpc_recovery.StreamServerInterceptor()),
+				grpc_middleware.WithUnaryServerChain(
+					grpc_recovery.UnaryServerInterceptor()),
+			},
+		},
+	}
+	suite.Run(t, s)
+}
+
+type RecoverySuite struct {
+	*grpc_testing.InterceptorTestSuite
+}
+
+func (s *RecoverySuite) TestUnary_SuccessfulRequest() {
+	_, err := s.Client.Ping(s.SimpleCtx(), goodPing)
+	require.NoError(s.T(), err, "no error must occur")
+}
+
+func (s *RecoverySuite) TestUnary_PanickingRequest() {
+	_, err := s.Client.Ping(s.SimpleCtx(), panicPing)
+	require.Error(s.T(), err, "there must be an error")
+	assert.Equal(s.T(), codes.Internal, grpc.Code(err), "must error with internal")
+	assert.Equal(s.T(), "very bad thing happened", grpc.ErrorDesc(err), "must error with message")
+}
+
+func (s *RecoverySuite) TestStream_SuccessfulReceive() {
+	stream, err := s.Client.PingList(s.SimpleCtx(), goodPing)
+	require.NoError(s.T(), err, "should not fail on establishing the stream")
+	pong, err := stream.Recv()
+	require.NoError(s.T(), err, "no error must occur")
+	require.NotNil(s.T(), pong, "pong must not be nil")
+}
+
+func (s *RecoverySuite) TestStream_PanickingReceive() {
+	stream, err := s.Client.PingList(s.SimpleCtx(), panicPing)
+	require.NoError(s.T(), err, "should not fail on establishing the stream")
+	_, err = stream.Recv()
+	require.Error(s.T(), err, "there must be an error")
+	assert.Equal(s.T(), codes.Internal, grpc.Code(err), "must error with internal")
+	assert.Equal(s.T(), "very bad thing happened", grpc.ErrorDesc(err), "must error with message")
+}
+
+func TestRecoveryOverrideSuite(t *testing.T) {
+	opts := []grpc_recovery.Option{
+		grpc_recovery.WithRecoveryHandler(func(p interface{}) (err error) {
+			return grpc.Errorf(codes.Unknown, "panic triggered: %v", p)
+		}),
+	}
+	s := &RecoveryOverrideSuite{
+		InterceptorTestSuite: &grpc_testing.InterceptorTestSuite{
+			TestService: &recoveryAssertService{TestServiceServer: &grpc_testing.TestPingService{T: t}},
+			ServerOpts: []grpc.ServerOption{
+				grpc_middleware.WithStreamServerChain(
+					grpc_recovery.StreamServerInterceptor(opts...)),
+				grpc_middleware.WithUnaryServerChain(
+					grpc_recovery.UnaryServerInterceptor(opts...)),
+			},
+		},
+	}
+	suite.Run(t, s)
+}
+
+type RecoveryOverrideSuite struct {
+	*grpc_testing.InterceptorTestSuite
+}
+
+func (s *RecoveryOverrideSuite) TestUnary_SuccessfulRequest() {
+	_, err := s.Client.Ping(s.SimpleCtx(), goodPing)
+	require.NoError(s.T(), err, "no error must occur")
+}
+
+func (s *RecoveryOverrideSuite) TestUnary_PanickingRequest() {
+	_, err := s.Client.Ping(s.SimpleCtx(), panicPing)
+	require.Error(s.T(), err, "there must be an error")
+	assert.Equal(s.T(), codes.Unknown, grpc.Code(err), "must error with unknown")
+	assert.Equal(s.T(), "panic triggered: very bad thing happened", grpc.ErrorDesc(err), "must error with message")
+}
+
+func (s *RecoveryOverrideSuite) TestStream_SuccessfulReceive() {
+	stream, err := s.Client.PingList(s.SimpleCtx(), goodPing)
+	require.NoError(s.T(), err, "should not fail on establishing the stream")
+	pong, err := stream.Recv()
+	require.NoError(s.T(), err, "no error must occur")
+	require.NotNil(s.T(), pong, "pong must not be nil")
+}
+
+func (s *RecoveryOverrideSuite) TestStream_PanickingReceive() {
+	stream, err := s.Client.PingList(s.SimpleCtx(), panicPing)
+	require.NoError(s.T(), err, "should not fail on establishing the stream")
+	_, err = stream.Recv()
+	require.Error(s.T(), err, "there must be an error")
+	assert.Equal(s.T(), codes.Unknown, grpc.Code(err), "must error with unknown")
+	assert.Equal(s.T(), "panic triggered: very bad thing happened", grpc.ErrorDesc(err), "must error with message")
+}

recovery/options.go

@@ -0,0 +1,32 @@
+// Copyright 2017 David Ackroyd. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+package grpc_recovery
+
+var (
+	defaultOptions = &options{
+		recoveryHandlerFunc: nil,
+	}
+)
+
+type options struct {
+	recoveryHandlerFunc RecoveryHandlerFunc
+}
+
+func evaluateOptions(opts []Option) *options {
+	optCopy := &options{}
+	*optCopy = *defaultOptions
+	for _, o := range opts {
+		o(optCopy)
+	}
+	return optCopy
+}
+
+type Option func(*options)
+
+// WithRecoveryHandler customizes the function for recovering from a panic.
+func WithRecoveryHandler(f RecoveryHandlerFunc) Option {
+	return func(o *options) {
+		o.recoveryHandlerFunc = f
+	}
+}