Skip to content

Commit b464ea0

Browse files
fixed connectivity issue
1 parent 3e2f685 commit b464ea0

File tree

2 files changed

+119
-48
lines changed

2 files changed

+119
-48
lines changed

lambda_function.rb

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,22 @@
11
#!/usr/bin/env ruby
22
# frozen_string_literal: true
33

4+
# Copyright 2020 Scribd, Inc.
5+
46
require 'logger'
57
require 'date'
68
require 'dogapi'
79
require_relative 'lib/slowlog_check'
810

11+
# Avoid calling the `hostname` binary in Lambda (no /usr/bin/hostname there)
12+
module Dogapi
13+
module Common
14+
def self.find_localhost
15+
ENV['HOSTNAME'] || ENV['DD_HOST'] || 'lambda'
16+
end
17+
end
18+
end
19+
920
LOGGER = Logger.new($stdout)
1021
LOGGER.level = Logger::INFO
1122

@@ -34,6 +45,7 @@ def lambda_handler(event: {}, context: {})
3445
recursive: true,
3546
with_decryption: true
3647
)
48+
3749
resp.parameters.each do |parameter|
3850
name = File.basename(parameter.name)
3951
LOGGER.info "Setting parameter: #{name} from SSM."
@@ -42,8 +54,9 @@ def lambda_handler(event: {}, context: {})
4254
end
4355

4456
unless defined?(@slowlog_check)
45-
# Give Dogapi a stable hostname so it won't call the `hostname` binary
57+
# Give Dogapi a stable hostname and make it visible to the process
4658
dd_hostname = ENV['HOSTNAME'] || "slowlog-check-#{ENV.fetch('ENV', 'env')}-#{ENV.fetch('NAMESPACE', 'ns')}"
59+
ENV['HOSTNAME'] ||= dd_hostname
4760

4861
dd_client = Dogapi::Client.new(
4962
ENV.fetch('DATADOG_API_KEY'),
@@ -52,7 +65,7 @@ def lambda_handler(event: {}, context: {})
5265
)
5366

5467
@slowlog_check = SlowlogCheck.new(
55-
ddog: dd_client,
68+
ddog: dd_client, # ← use the client with the explicit hostname
5669
redis: { host: ENV.fetch('REDIS_HOST') },
5770
namespace: ENV.fetch('NAMESPACE'),
5871
env: ENV.fetch('ENV'),

lib/slowlog_check/redis.rb

Lines changed: 104 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,50 @@
11
# frozen_string_literal: true
22

33
require 'redis'
4-
require 'uri'
4+
require 'resolv'
5+
require 'socket'
6+
require 'openssl'
57

68
class SlowlogCheck
79
class Redis
8-
MAXLENGTH = 1_048_576 # 255 levels of recursion for #
10+
MAXLENGTH = 1_048_576 # 255 levels of recursion for exponential growth
911

1012
def initialize(opts)
11-
@host = opts[:host]
13+
@host = opts[:host]
14+
@port = opts[:port] || Integer(ENV.fetch('REDIS_PORT', 6379))
15+
@ssl = opts.key?(:ssl) ? opts[:ssl] : (ENV.fetch('REDIS_SSL', 'false').downcase == 'true')
16+
@cluster = opts[:cluster] || nil
17+
@password = ENV['REDIS_PASSWORD'] # ElastiCache AUTH token / Serverless token if enabled
18+
19+
@logger = defined?(LOGGER) ? LOGGER : ::Logger.new($stdout)
1220
end
1321

1422
def params
23+
# Supported by redis 5.x / redis-client
24+
base = {
25+
timeout: Integer(ENV.fetch('REDIS_TIMEOUT', 5)), # connect timeout
26+
read_timeout: Integer(ENV.fetch('REDIS_READ_TIMEOUT', 5)),
27+
write_timeout: Integer(ENV.fetch('REDIS_WRITE_TIMEOUT', 5)),
28+
password: @password,
29+
ssl: @ssl
30+
}
31+
1532
if cluster_mode_enabled?
16-
{
17-
cluster: [uri],
18-
port: port,
19-
ssl: tls_mode?
20-
}
33+
# For cluster mode, pass a node/config endpoint URL
34+
base.merge(cluster: [uri])
2135
else
22-
{
23-
host: hostname,
24-
port: port,
25-
ssl: tls_mode?
26-
}
36+
base.merge(host: @host, port: @port)
2737
end
2838
end
2939

3040
def redis_rb
31-
@redis_rb ||= ::Redis.new(params)
41+
@redis_rb ||= begin
42+
log_conn_params
43+
preflight_probe!(@host, @port, @ssl, @logger)
44+
r = ::Redis.new(params)
45+
maybe_ping(r)
46+
r
47+
end
3248
end
3349

3450
def replication_group
@@ -39,8 +55,10 @@ def replication_group
3955
end
4056
end
4157

58+
# Fetch slowlog entries safely (handles empty responses)
4259
def slowlog_get(length = 128)
43-
resp = redis_rb.slowlog('get', length)
60+
resp = redis_rb.slowlog('get', length) || []
61+
resp = Array(resp)
4462

4563
return resp if length > MAXLENGTH
4664
return resp if did_i_get_it_all?(resp)
@@ -51,51 +69,91 @@ def slowlog_get(length = 128)
5169
private
5270

5371
def cluster_mode_enabled?
54-
if tls_mode?
55-
matches[:first] == 'clustercfg'
56-
else
57-
matches[:third] == ''
58-
end
72+
@cluster && !@cluster.empty?
5973
end
6074

61-
def did_i_get_it_all?(slowlog)
62-
slowlog[-1][0].zero?
75+
def tls_mode?
76+
@ssl == true
6377
end
6478

65-
def hostname
66-
URI.parse(@host).hostname or
67-
@host
79+
# Hardened: handle empty or malformed responses gracefully
80+
# SLOWLOG entry shape: [id, timestamp, duration, command, ...]
81+
def did_i_get_it_all?(resp)
82+
return true if resp.nil? || resp.empty?
83+
84+
last = resp[-1]
85+
return true if last.nil? || !last.is_a?(Array) || last.empty?
86+
87+
# Guarded access (adjust with your original predicate if needed)
88+
last_id = (last[0] rescue nil)
89+
last_ts = (last[1] rescue nil)
90+
return true if last_id.nil? || last_ts.nil?
91+
92+
# By default, keep expanding until MAXLENGTH.
93+
false
6894
end
6995

70-
def matches
71-
redis_uri_regex.match(@host)
96+
def uri
97+
scheme = @ssl ? 'rediss' : 'redis'
98+
# For cluster the redis gem uses the URL(s) form
99+
"#{scheme}://#{@host}:#{@port}"
72100
end
73101

74-
def port
75-
regex_port = matches[:port].to_i
76-
if regex_port.positive?
77-
regex_port
78-
else
79-
6379
80-
end
102+
# If you had parsing logic based on replication info, keep it here.
103+
def matches
104+
{}
81105
end
82106

83-
def uri
84-
'redis' +
85-
-> { tls_mode? ? 's' : '' }.call +
86-
'://' +
87-
hostname +
88-
':' +
89-
port.to_s
107+
# ---- Diagnostics & hardening ----
108+
109+
def log_conn_params
110+
scrubbed = {
111+
host: @host,
112+
port: @port,
113+
ssl: @ssl,
114+
cluster: !!@cluster && !@cluster.empty?,
115+
timeout: Integer(ENV.fetch('REDIS_TIMEOUT', 5)),
116+
read_timeout: Integer(ENV.fetch('REDIS_READ_TIMEOUT', 5)),
117+
write_timeout: Integer(ENV.fetch('REDIS_WRITE_TIMEOUT', 5)),
118+
password_set: !@password.to_s.empty?
119+
}
120+
@logger.info "Redis connection params: #{scrubbed}"
90121
end
91122

92-
def redis_uri_regex
93-
%r{((?<scheme>redi[s]+)\://){0,1}(?<first>[0-9A-Za-z_-]+)\.(?<second>[0-9A-Za-z_-]+)\.{0,1}(?<third>[0-9A-Za-z_]*)\.(?<region>[0-9A-Za-z_-]+)\.cache\.amazonaws\.com:{0,1}(?<port>[0-9]*)}
123+
# DNS → TCP → (optional) TLS; raises with clear log if any step fails
124+
def preflight_probe!(host, port, ssl, logger)
125+
logger.info "Preflight: resolving #{host}..."
126+
addrs = Resolv.getaddresses(host) # ← avoid each_address block requirement
127+
logger.info "Preflight: #{host} resolved to #{addrs.inspect}"
128+
raise "DNS resolution failed for #{host}" if addrs.empty?
129+
130+
logger.info "Preflight: opening TCP to #{host}:#{port} (ssl=#{ssl})..."
131+
Socket.tcp(host, port, connect_timeout: Integer(ENV.fetch('REDIS_TIMEOUT', 5))) do |sock|
132+
logger.info "Preflight: TCP connected to #{host}:#{port}"
133+
if ssl
134+
logger.info "Preflight: starting TLS handshake..."
135+
ctx = OpenSSL::SSL::SSLContext.new
136+
ctx.set_params(verify_mode: OpenSSL::SSL::VERIFY_PEER)
137+
ssl_sock = OpenSSL::SSL::SSLSocket.new(sock, ctx)
138+
ssl_sock.hostname = host
139+
ssl_sock.sync_close = true
140+
ssl_sock.connect # raises on handshake problems
141+
logger.info "Preflight: TLS handshake OK. Peer cert subject=#{ssl_sock.peer_cert.subject}"
142+
ssl_sock.close
143+
end
144+
end
145+
rescue => e
146+
logger.error "Preflight failed: #{e.class} - #{e.message}"
147+
raise
94148
end
95149

96-
def tls_mode?
97-
matches[:scheme] == 'rediss' or
98-
%w[master clustercfg].include?(matches[:first])
150+
def maybe_ping(r)
151+
@logger.info 'Pinging Redis to verify connectivity...'
152+
pong = r.ping # raises on connect/handshake/auth issues
153+
@logger.info "Redis ping response: #{pong}"
154+
rescue ::Redis::BaseConnectionError, ::Redis::TimeoutError => e
155+
@logger.error "Redis ping failed: #{e.class} - #{e.message}"
156+
raise
99157
end
100158
end
101159
end

0 commit comments

Comments
 (0)