Skip to content

Commit

Permalink
Merge pull request #119 from oliver006/oh_cluster_tests
Browse files Browse the repository at this point in the history
Add cluster tests
  • Loading branch information
oliver006 authored Mar 22, 2018
2 parents 09e179c + 7d230f3 commit ed524c8
Show file tree
Hide file tree
Showing 7 changed files with 439 additions and 146 deletions.
27 changes: 16 additions & 11 deletions circle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,27 +6,32 @@ machine:
- /usr/bin/redis-server --port 6380:
background: true
environment:
SRC_LOCATION: "/home/ubuntu/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME"
COVERAGE_PROFILE: "/home/ubuntu/coverage.out"
GO_LDFLAGS: '-extldflags "-static" -X main.VERSION=$CIRCLE_TAG -X main.COMMIT_SHA1=$CIRCLE_SHA1 -X main.BUILD_DATE=$(date +%F-%T)'
MY_GO_VERSION: "1.9.2"
SRC_LOCATION: "/home/ubuntu/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME"
COVERAGE_PROFILE: "/home/ubuntu/coverage.out"
GO_LDFLAGS: '-extldflags "-static" -X main.VERSION=$CIRCLE_TAG -X main.COMMIT_SHA1=$CIRCLE_SHA1 -X main.BUILD_DATE=$(date +%F-%T)'
MY_GO_VERSION: "1.9.4"
REDIS_TEST_VERSION: "3.2.11"

dependencies:
pre:
- rm -rf /home/ubuntu/.go_workspace
- rm -rf /home/ubuntu/.go_project
- sudo service redis-server stop
- >
cd ~ && if [ ! -d "redis-3.2.10" ]; then
wget http://download.redis.io/releases/redis-3.2.10.tar.gz
tar xzf redis-3.2.10.tar.gz
cd redis-3.2.10 && make;
cd ~ && if [ ! -d "redis-$REDIS_TEST_VERSION" ]; then
wget http://download.redis.io/releases/redis-$REDIS_TEST_VERSION.tar.gz
tar xzf redis-$REDIS_TEST_VERSION.tar.gz
cd redis-$REDIS_TEST_VERSION && make;
fi
- cd ~/redis-3.2.10 && sudo make install
- cd ~/redis-$REDIS_TEST_VERSION && sudo make install
- sudo sed -i 's/bin/local\/bin/g' /etc/init/redis-server.conf
- sudo service redis-server start
#
# the next line will bring up a cluster of redis instances with slaves
# for addtl tests
- docker run -p 7000-7006:7000-7006 --name redis_cluster_test -d grokzen/redis-cluster
cache_directories:
- ~/redis-3.2.10
- ~/redis-$REDIS_TEST_VERSION
override:
- mkdir -p "/home/ubuntu/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME"
- ln -s $HOME/$CIRCLE_PROJECT_REPONAME $SRC_LOCATION
Expand All @@ -41,7 +46,7 @@ test:
- go get github.com/mattn/goveralls
- go get golang.org/x/tools/cmd/cover
override:
- cd $SRC_LOCATION && go test -v -covermode=atomic -cover -race -coverprofile=$COVERAGE_PROFILE ./exporter/...
- cd $SRC_LOCATION && TEST_REDIS_CLUSTER_MASTER_URI=localhost:7000 TEST_REDIS_CLUSTER_SLAVE_URI=localhost:7005 go test -v -covermode=atomic -cover -race -coverprofile=$COVERAGE_PROFILE ./exporter/...
post:
- if [ -n "$COVERALLS_TOKEN" ]; then /home/ubuntu/.go_workspace/bin/goveralls -coverprofile=$COVERAGE_PROFILE -service=circle-ci -repotoken=$COVERALLS_TOKEN ; fi

Expand Down
4 changes: 4 additions & 0 deletions contrib/sample_redis_hosts_file.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
redis://localhost:6379

redis://localhost:7000,password,alias
redis://localhost:7000,second-pwd
99 changes: 99 additions & 0 deletions exporter/discovery.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package exporter

import (
"encoding/csv"
"os"
"strings"

"github.com/cloudfoundry-community/go-cfenv"
log "github.com/sirupsen/logrus"
)

// loadRedisArgs loads the configuration for which redis hosts to monitor from either
// the environment or as passed from program arguments. Returns the list of host addrs,
// passwords, and their aliases.
func LoadRedisArgs(addr, password, alias, separator string) ([]string, []string, []string) {
if addr == "" {
addr = "redis://localhost:6379"
}
addrs := strings.Split(addr, separator)
passwords := strings.Split(password, separator)
for len(passwords) < len(addrs) {
passwords = append(passwords, passwords[0])
}
aliases := strings.Split(alias, separator)
for len(aliases) < len(addrs) {
aliases = append(aliases, aliases[0])
}
return addrs, passwords, aliases
}

// loadRedisFile opens the specified file and loads the configuration for which redis
// hosts to monitor. Returns the list of hosts addrs, passwords, and their aliases.
func LoadRedisFile(fileName string) ([]string, []string, []string, error) {
var addrs []string
var passwords []string
var aliases []string
file, err := os.Open(fileName)
if err != nil {
return nil, nil, nil, err
}
r := csv.NewReader(file)
r.FieldsPerRecord = -1
records, err := r.ReadAll()
if err != nil {
return nil, nil, nil, err
}
file.Close()
// For each line, test if it contains an optional password and alias and provide them,
// else give them empty strings
for _, record := range records {
length := len(record)
switch length {
case 3:
addrs = append(addrs, record[0])
passwords = append(passwords, record[1])
aliases = append(aliases, record[2])
case 2:
addrs = append(addrs, record[0])
passwords = append(passwords, record[1])
aliases = append(aliases, "")
case 1:
addrs = append(addrs, record[0])
passwords = append(passwords, "")
aliases = append(aliases, "")
}
}
return addrs, passwords, aliases, nil
}

func GetCloudFoundryRedisBindings() (addrs, passwords, aliases []string) {
if !cfenv.IsRunningOnCF() {
return
}

appEnv, err := cfenv.Current()
if err != nil {
log.Warnln("Unable to get current CF environment", err)
return
}

redisServices, err := appEnv.Services.WithTag("redis")
if err != nil {
log.Warnln("Error while getting redis services", err)
return
}

for _, redisService := range redisServices {
credentials := redisService.Credentials
addr := credentials["hostname"].(string) + ":" + credentials["port"].(string)
password := credentials["password"].(string)
alias := redisService.Name

addrs = append(addrs, addr)
passwords = append(passwords, password)
aliases = append(aliases, alias)
}

return
}
103 changes: 103 additions & 0 deletions exporter/discovery_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package exporter

import (
"log"
"testing"
)

func cmpStringArrays(a1, a2 []string) bool {
if len(a1) != len(a2) {
return false
}
for n := range a1 {
if a1[n] != a2[n] {
return false
}
}
return true
}

func TestLoadRedisArgs(t *testing.T) {
log.Println("TestLoadRedisArgs()")
tests := []struct {
addr, pwd, alias, sep string
wantAddr, wantPwds, wantAliases []string
}{
{
addr: "",
sep: ",",
wantAddr: []string{"redis://localhost:6379"},
wantPwds: []string{""},
wantAliases: []string{""},
},
{
addr: "redis://localhost:6379",
sep: ",",
wantAddr: []string{"redis://localhost:6379"},
wantPwds: []string{""},
wantAliases: []string{""},
},
{
addr: "redis://localhost:6379,redis://localhost:7000",
sep: ",",
wantAddr: []string{"redis://localhost:6379", "redis://localhost:7000"},
wantPwds: []string{"", ""},
wantAliases: []string{"", ""},
},
{
addr: "redis://localhost:6379,redis://localhost:7000,redis://localhost:7001",
sep: ",",
wantAddr: []string{"redis://localhost:6379", "redis://localhost:7000", "redis://localhost:7001"},
wantPwds: []string{"", "", ""},
wantAliases: []string{"", "", ""},
},
{
alias: "host-1",
sep: ",",
wantAddr: []string{"redis://localhost:6379"},
wantPwds: []string{""},
wantAliases: []string{"host-1"},
},
}

for _, test := range tests {
sep := test.sep
addrs, pwds, aliases := LoadRedisArgs(test.addr, test.pwd, test.alias, sep)
if !cmpStringArrays(addrs, test.wantAddr) {
t.Errorf("addrs not matching wantAliases, got: %v want: %v", addrs, test.wantAddr)
}
if !cmpStringArrays(pwds, test.wantPwds) {
t.Errorf("pwds not matching wantAliases, got: %v want: %v", pwds, test.wantPwds)
}
if !cmpStringArrays(aliases, test.wantAliases) {
t.Errorf("aliases not matching wantAliases, got: %v want: %v", aliases, test.wantAliases)
}
}
}

func TestLoadRedisFile(t *testing.T) {
if _, _, _, err := LoadRedisFile("doesnt-exist.txt"); err == nil {
t.Errorf("should have failed opening non existing file")
return
}

addrs, pwds, aliases, err := LoadRedisFile("../contrib/sample_redis_hosts_file.txt")
if err != nil {
t.Errorf("LoadRedisFile() failed, err: %s", err)
return
}
log.Printf("aliases: %v \n", aliases)
if !cmpStringArrays(addrs, []string{"redis://localhost:6379", "redis://localhost:7000", "redis://localhost:7000"}) {
t.Errorf("addrs not matching want")
}
if !cmpStringArrays(pwds, []string{"", "password", "second-pwd"}) {
t.Errorf("pwds not matching want")
}
if !cmpStringArrays(aliases, []string{"", "alias", ""}) {
t.Errorf("aliases not matching want")
}
}

func TestGetCloudFoundryRedisBindings(t *testing.T) {
GetCloudFoundryRedisBindings()
}
33 changes: 22 additions & 11 deletions exporter/redis.go
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,16 @@ func (e *Exporter) extractInfoMetrics(info, addr string, alias string, scrapes c
return nil
}

func doRedisCmd(c redis.Conn, cmd string, args ...interface{}) (reply interface{}, err error) {
log.Debugf("c.Do() - running command: %s %s", cmd, args)
defer log.Debugf("c.Do() - done")
res, err := c.Do(cmd, args...)
if err != nil {
log.Debugf("c.Do() - err: %s", err)
}
return res, err
}

func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx int) error {
options := []redis.DialOption{
redis.DialConnectTimeout(5 * time.Second),
Expand All @@ -596,7 +606,7 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
}

if err != nil {
log.Printf("redis err: %s", err)
log.Debugf("aborting for addr: %s - redis err: %s", addr, err)
return err
}

Expand All @@ -615,24 +625,24 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
log.Debugf("Redis CONFIG err: %s", err)
}

info, err := redis.String(c.Do("INFO", "ALL"))
info, err := redis.String(doRedisCmd(c, "INFO", "ALL"))
if err == nil {
e.extractInfoMetrics(info, addr, e.redis.Aliases[idx], scrapes, dbCount, true)
} else {
log.Errorf("Redis INFO err: %s", err)
return err
}

if strings.Index(info, "cluster_enabled:1") != -1 {
info, err = redis.String(c.Do("CLUSTER", "INFO"))
if strings.Contains(info, "cluster_enabled:1") {
info, err = redis.String(doRedisCmd(c, "CLUSTER", "INFO"))
if err != nil {
log.Errorf("redis err: %s", err)
} else {
e.extractInfoMetrics(info, addr, e.redis.Aliases[idx], scrapes, dbCount, false)
}
}

if reply, err := c.Do("LATENCY", "LATEST"); err == nil {
if reply, err := doRedisCmd(c, "LATENCY", "LATEST"); err == nil {
var eventName string
var spikeLast, milliseconds, max int64
if tempVal, _ := reply.([]interface{}); len(tempVal) > 0 {
Expand All @@ -646,13 +656,14 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
}
}

log.Debugf("e.keys: %#v", e.keys)
for _, k := range e.keys {
if _, err := c.Do("SELECT", k.db); err != nil {
if _, err := doRedisCmd(c, "SELECT", k.db); err != nil {
continue
}

obtainedKeys := []string{}
if tempVal, err := redis.Strings(c.Do("KEYS", k.key)); err == nil && tempVal != nil {
if tempVal, err := redis.Strings(doRedisCmd(c, "KEYS", k.key)); err == nil && tempVal != nil {
for _, tempKey := range tempVal {
log.Debugf("Append result: %s", tempKey)
obtainedKeys = append(obtainedKeys, tempKey)
Expand All @@ -662,7 +673,7 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
for _, key := range obtainedKeys {
dbLabel := "db" + k.db
keyLabel := key
if tempVal, err := c.Do("GET", key); err == nil && tempVal != nil {
if tempVal, err := doRedisCmd(c, "GET", key); err == nil && tempVal != nil {
if val, err := strconv.ParseFloat(fmt.Sprintf("%s", tempVal), 64); err == nil {
e.keyValues.WithLabelValues(addr, e.redis.Aliases[idx], dbLabel, keyLabel).Set(val)
}
Expand All @@ -676,26 +687,26 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
"PFCOUNT",
"STRLEN",
} {
if tempVal, err := c.Do(op, key); err == nil && tempVal != nil {
if tempVal, err := doRedisCmd(c, op, key); err == nil && tempVal != nil {
e.keySizes.WithLabelValues(addr, e.redis.Aliases[idx], dbLabel, keyLabel).Set(float64(tempVal.(int64)))
break
}
}
}
}

log.Debugf("scrapeRedisHost() done")
return nil
}

func (e *Exporter) scrape(scrapes chan<- scrapeResult) {

defer close(scrapes)

now := time.Now().UnixNano()
e.totalScrapes.Inc()

errorCount := 0
for idx, addr := range e.redis.Addrs {

var up float64 = 1
if err := e.scrapeRedisHost(scrapes, addr, idx); err != nil {
errorCount++
Expand Down
Loading

0 comments on commit ed524c8

Please sign in to comment.