Merge pull request #13 from Azure-Samples/ethanperry/introduce-tools

Ethanperry/introduce tools

.github/workflows/docker_build.yml

@@ -10,7 +10,6 @@ jobs:
   docker_build:
     name: 'Build docker containers'
     runs-on: ubuntu-latest
-    environment: production
 
     defaults:
       run:

.gitignore

@@ -396,7 +396,3 @@ FodyWeavers.xsd
 
 # JetBrains Rider
 *.sln.iml
-
-bin
-cover.tmp.out
-coverage.out

go.mod → lib/mage/go.mod

@@ -1,4 +1,4 @@
-module github.com/Azure-Samples/aio-dev-toolbox/toolbox
+module github.com/Azure-Samples/explore-iot-operations/lib/mage
 
 go 1.21.3
 

samples/callout/.gitignore

@@ -0,0 +1,24 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+bin/logs
+
+# Go workspace file
+go.work
+
+bin

samples/callout/.vscode/launch.json

@@ -0,0 +1,19 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch",
+            "type": "go",
+            "request": "launch",
+            "mode": "debug",
+            "cwd": "${workspaceRoot}/bin",
+            "program": "${workspaceRoot}",
+            "env": {},
+            "args": [],
+            "showLog": true
+        }
+    ]
+}

samples/callout/Dockerfile

@@ -0,0 +1,23 @@
+# syntax=docker/dockerfile:1
+
+FROM golang:1.21-alpine
+
+# Set destination for COPY
+WORKDIR /app
+ADD . /app
+
+# Download Go modules
+RUN go mod download
+
+# Build
+RUN CGO_ENABLED=0 GOOS=linux go build -o /callout
+
+# Optional:
+# To bind to a TCP port, runtime parameters must be supplied to the docker command.
+# But we can document in the Dockerfile what ports
+# the application is going to listen on by default.
+# https://docs.docker.com/engine/reference/builder/#expose
+EXPOSE 8080
+
+# Run
+CMD ["/callout"]

samples/callout/README.md

@@ -0,0 +1,76 @@
+# callout
+This is a HTTP endpoint that can be used to debug or process data in AIO Data Processor using **Call out HTTP**.
+
+## Usage
+
+### Deploy container to Kubernetes cluster
+
+This utility can be deployed as a service in your Kubernetes cluster. You can use the container that is published to ACR(Azure Container Registry) using [setup/service.yaml](setup/service.yaml) . 
+```
+kubectl apply -f setup/service.yaml
+```
+
+### Debugging pipeline
+Data Processor has a **Call out HTTP** stage where you can call a HTTP endpoint from with in the pipeline. In that callout stage, you can use the api/echo route to print the contents of the message. Where ever you need to see the message, you can add a callout stage.
+
+|Parameter | Value       | Description  |
+|----------|-------------|--------------|
+| Method   | GET or POST | any payload sent in the body is printed as pretty JSON |
+| URL      | http://callout.default.svc.cluster.local/api/echo/myStage | The URL of the callout endpoint hosted in the cluster. To disambiguate the print outputs, you can use a string like *myStage* or *stage2* etc. |
+
+### Quality factor
+You can compute quality factor using a **Call out HTTP** stage hitting this HTTP endpoint. In that callout stage, you can use the api/qfactor route to comput qFactor, Quality and shift.
+
+|Parameter | Value       | Description  |
+|----------|-------------|--------------|
+| Method   | POST |  |
+| URL      | http://callout.default.svc.cluster.local/api/qfactor |  |
+
+#### Input Message ####
+```JSON
+{
+  "Payload": {
+    "age": 14,
+    "asset_id": "Red_S1",
+    "asset_name": "Redmond_Slicer_Redmond_Slicer__asset_0",
+    "country": "USA",
+    "humidity": 94.49016579867568,
+    "id": "Red_S1",
+    "machine_status": 0,
+    "operating_time": 12527,
+    "product": "Takis",
+    "site": "Redmond",
+    "source_timestamp": "2023-10-18T18:07:45.575Z",
+    "temperature": 91.06476575011023,
+    "vibration": 45.53238287505511
+  },
+  "SequenceNumber": 12515,
+  "Timestamp": "2023-10-18T11:07:45.566556393-07:00"
+}
+```
+#### Output Message ####
+```JSON
+{
+  "Payload": {
+    "age": 14,
+    "asset_id": "Red_S1",
+    "asset_name": "Redmond_Slicer_Redmond_Slicer__asset_0",
+    "country": "USA",
+    "humidity": 94.49016579867568,
+    "id": "Red_S1",
+    "machine_status": 0,
+    "operating_time": 12527,
+    "pressure": 0,
+    "product": "Takis",
+    "site": "Redmond",
+    "temperature": 91.06476575011023,
+    "vibration": 45.53238287505511,
+    "q_factor": 0.8,
+    "quality": "Good",
+    "shift": 3,
+    "source_timestamp": "2023-10-18T18:07:45.575Z"
+  },
+  "SequenceNumber": 12515,
+  "Timestamp": "2023-10-18T11:07:45.566556393-07:00"
+}
+```

samples/callout/cmd/config.go

@@ -0,0 +1,23 @@
+package main
+
+type (
+	Config struct {
+		LogLevel string `json:"logLevel"` // logging level for the application
+		LogsDir  string `json:"logsDir"`  // directory into which logs are written
+	}
+
+	config struct {
+		Logger Config `json:"logger"`
+		Port   int    `json:"port"`
+	}
+)
+
+func newConfig() *config {
+	return &config{
+		Logger: Config{
+			LogLevel: "Debug",
+			LogsDir:  "./logs",
+		},
+		Port: 8888,
+	}
+}

samples/callout/cmd/main.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"context"
+	_ "embed"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"path"
+	"strings"
+
+	"github.com/reddydMSFT/callout/pkg/serving"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/viper"
+	"gopkg.in/natefinch/lumberjack.v2"
+)
+
+func main() {
+	// handle process exit gracefully
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, os.Interrupt)
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+
+	defer func() {
+		// Close the os signal channel to prevent any leak.
+		signal.Stop(sig)
+	}()
+
+	// load configuration and initialize logger
+	cfg, err := loadConfig()
+	if err != nil {
+		panic(fmt.Errorf("failed to initialize configuration. %w", err))
+	}
+	initLogger(cfg)
+
+	go serving.StartAdmin(cfg.Port)
+
+	// Wait signal / cancellation
+	<-sig
+
+	cancel() // Wait for device to completely shut down.
+}
+
+// loadConfig loads the configuration file
+func loadConfig() (*config, error) {
+	colorReset := "\033[0m"
+	//colorRed := "\033[31m"
+	colorGreen := "\033[32m"
+	//colorYellow := "\033[33m"
+	colorBlue := "\033[34m"
+	//colorPurple := "\033[35m"
+	//colorCyan := "\033[36m"
+	//colorWhite := "\033[37m"
+	fmt.Printf(string(colorGreen))
+	fmt.Printf(`
+ ██████╗ █████╗ ██╗     ██╗      ██████╗ ██╗   ██╗████████╗
+██╔════╝██╔══██╗██║     ██║     ██╔═══██╗██║   ██║╚══██╔══╝
+██║     ███████║██║     ██║     ██║   ██║██║   ██║   ██║   
+██║     ██╔══██║██║     ██║     ██║   ██║██║   ██║   ██║   
+╚██████╗██║  ██║███████╗███████╗╚██████╔╝╚██████╔╝   ██║   
+ ╚═════╝╚═╝  ╚═╝╚══════╝╚══════╝ ╚═════╝  ╚═════╝    ╚═╝   
+`)
+	fmt.Printf(string(colorBlue))
+	fmt.Printf("        AIO DATA PROCESSOR CALLOUT\n")
+	fmt.Printf(string(colorReset))
+
+	viper.SetConfigName("callout")
+	viper.SetConfigType("json")
+	viper.AddConfigPath(".")
+	viper.AddConfigPath("./bin")
+
+	viper.AutomaticEnv()
+	if err := viper.ReadInConfig(); err != nil {
+		if _, ok := err.(viper.ConfigFileNotFoundError); ok {
+			fmt.Print(`Add a configuration file (callout.json) with the file contents below:
+
+{
+  "logger": {
+    "logLevel": "Debug",
+    "logsDir": "./logs"
+  },
+  "port": 8888
+}
+
+\n`)
+			return nil, err
+		}
+	}
+
+	cfg := newConfig()
+	if err := viper.Unmarshal(cfg); err != nil {
+		return nil, err
+	}
+
+	//fmt.Printf("loaded configuration from %s\n", viper.ConfigFileUsed())
+	return cfg, nil
+}
+
+// initLogger initializes the logger with output format
+func initLogger(cfg *config) {
+	var writers []io.Writer
+	writers = append(writers, zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: "15:04:05"})
+
+	fileLoggingEnabled := false
+	if len(cfg.Logger.LogsDir) > 0 {
+		fileLoggingEnabled = true
+	}
+	if fileLoggingEnabled {
+		logsDir := cfg.Logger.LogsDir
+		if err := os.MkdirAll(logsDir, 0744); err != nil {
+			fmt.Printf("can't create log directory, so file logging is disabled, error: %s", err.Error())
+		} else {
+			fileWriter := &lumberjack.Logger{
+				Filename:   path.Join(logsDir, "callout.log"),
+				MaxBackups: 3,  // files
+				MaxSize:    10, // megabytes
+				MaxAge:     30, // days
+			}
+
+			writers = append(writers, fileWriter)
+			//fmt.Printf("file logging is enabled, logsDir: %s\n", logsDir)
+		}
+	}
+	mw := io.MultiWriter(writers...)
+
+	log.Logger = zerolog.New(mw).With().Timestamp().Logger()
+	//log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: "15:04:05"})
+
+	switch strings.ToLower(cfg.Logger.LogLevel) {
+	case "panic":
+		zerolog.SetGlobalLevel(zerolog.PanicLevel)
+	case "fatal":
+		zerolog.SetGlobalLevel(zerolog.FatalLevel)
+	case "error":
+		zerolog.SetGlobalLevel(zerolog.ErrorLevel)
+	case "warn":
+		zerolog.SetGlobalLevel(zerolog.WarnLevel)
+	case "info":
+		zerolog.SetGlobalLevel(zerolog.InfoLevel)
+	case "trace":
+		zerolog.SetGlobalLevel(zerolog.TraceLevel)
+	default:
+		zerolog.SetGlobalLevel(zerolog.DebugLevel)
+	}
+}