Merge pull request #48 from factorysh/features/logs

Expose logs using REST

Author: athoune

application/application.go

@@ -152,6 +152,7 @@ func New(cfg *conf.Conf) (*Application, error) {
 						r.Post("/", a.PostTaskHandler)
 						r.Get("/", a.TaskHandler(false))
 						r.Get("/volumes/*", a.VolumesHandler(6, false))
+						r.Get("/logs", a.TaskLogsHandler(false))
 					})
 					r.Group(func(r chi.Router) {
 						r.Use(a.RefererMiddleware)
@@ -164,6 +165,7 @@ func New(cfg *conf.Conf) (*Application, error) {
 						r.Use(authMiddleware.Middleware())
 						r.Get("/", a.TaskHandler(true))
 						r.Get("/volumes/*", a.VolumesHandler(6, true))
+						r.Get("/logs", a.TaskLogsHandler(true))
 					})
 					r.Group(func(r chi.Router) {
 						r.Use(a.RefererMiddleware)

application/application_test.go

@@ -188,6 +188,19 @@ func TestApplication(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Contains(t, string(data), "Bob")
 
+	req, err = mkRequest(key)
+	assert.NoError(t, err)
+	req.Method = http.MethodGet
+	req.URL, err = url.Parse(fmt.Sprintf("%s/service/demo/group/project/-/master/%s/logs", srvApp.URL, mockupCommit))
+	assert.NoError(t, err)
+	r, err = cli.Do(req)
+	assert.NoError(t, err)
+	defer r.Body.Close()
+	assert.Equal(t, 200, r.StatusCode)
+	data, err = ioutil.ReadAll(r.Body)
+	assert.NoError(t, err)
+	assert.Contains(t, string(data), "Bob")
+
 }
 
 func TestApplicationStart(t *testing.T) {

application/task.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/docker/docker/pkg/stdcopy"
 	_claims "github.com/factorysh/microdensity/claims"
 	"github.com/factorysh/microdensity/task"
 	"github.com/go-chi/chi/v5"
@@ -103,12 +104,12 @@ func (a *Application) addTask(t *task.Task, args map[string]string) error {
 		return err
 	}
 
-	err = a.storage.Upsert(t)
+	err = a.queue.Put(t, args)
 	if err != nil {
 		return err
 	}
 
-	err = a.queue.Put(t, args)
+	err = a.storage.Upsert(t)
 	if err != nil {
 		return err
 	}
@@ -180,3 +181,52 @@ func (a *Application) TaskIDHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 }
+
+// TaskLogsHandler get a logs for a task
+func (a *Application) TaskLogsHandler(latest bool) func(http.ResponseWriter, *http.Request) {
+
+	return func(w http.ResponseWriter, r *http.Request) {
+		l := a.logger.With(
+			zap.String("url", r.URL.String()),
+			zap.String("service", chi.URLParam(r, "serviceID")),
+			zap.String("project", chi.URLParam(r, "project")),
+			zap.String("branch", chi.URLParam(r, "branch")),
+			zap.String("commit", chi.URLParam(r, "commit")),
+		)
+
+		t, err := a.storage.GetByCommit(
+			chi.URLParam(r, "serviceID"),
+			chi.URLParam(r, "project"),
+			chi.URLParam(r, "branch"),
+			chi.URLParam(r, "commit"),
+			latest,
+		)
+
+		if err != nil {
+			l.Warn("Task get error", zap.Error(err))
+			w.WriteHeader(http.StatusNotFound)
+			w.Write([]byte(http.StatusText(http.StatusNotFound)))
+			return
+		}
+
+		reader, err := t.Logs(r.Context(), false)
+		if err != nil {
+			l.Warn("Task log error", zap.Error(err))
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(http.StatusText(http.StatusInternalServerError)))
+			return
+		}
+
+		// just stdout for now
+		// kudos @ndeloof, @rumpl, @glours
+		_, err = stdcopy.StdCopy(w, nil, reader)
+		if err != nil {
+			l.Error("Task log stdcopy write error", zap.Error(err))
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(http.StatusText(http.StatusInternalServerError)))
+		}
+
+		w.Header().Set("Content-Type", "text/plain")
+	}
+
+}

queue/queue.go

@@ -71,11 +71,13 @@ func (q *Queue) Put(item *task.Task, env map[string]string) error {
 	q.Lock()
 	defer q.Unlock()
 
-	err := q.runner.Prepare(item, env)
+	runnable, err := q.runner.Prepare(item, env)
 	if err != nil {
 		return err
 	}
 
+	item.Run = runnable
+
 	q.items.Enqueue(item)
 
 	queueAdded.Inc()

run/compose.go

@@ -214,7 +214,6 @@ func (c *ComposeRun) runCommand(stdout io.WriteCloser, stderr io.WriteCloser, co
 		Force: true,
 	})
 
-	l.Info("Run service")
 	n, err := c.service.RunOneOffContainer(c.runCtx, c.project, api.RunOptions{
 		Name:       fmt.Sprintf("%s_%s_%v", c.project.Name, c.run, c.id),
 		Service:    c.run,

run/compose_test.go

@@ -85,5 +85,4 @@ func TestCompose(t *testing.T) {
 	out, err = ioutil.ReadAll(buff)
 	assert.NoError(t, err)
 	assert.True(t, strings.HasPrefix(string(out), "8.8.8.8"))
-
 }

run/run.go

@@ -48,26 +48,26 @@ func NewRunner(servicesDir string, volumesRoot string, hosts []string) (*Runner,
 // Prepare the run
 // Prepare is synchronous, in order to raise an error in the REST endpoint.
 // Prepare checks volumes stuff.
-func (r *Runner) Prepare(t *task.Task, env map[string]string) error {
+func (r *Runner) Prepare(t *task.Task, env map[string]string) (string, error) {
 	if t.Id == uuid.Nil {
-		return fmt.Errorf("task requires an ID to be prepared")
+		return "", fmt.Errorf("task requires an ID to be prepared")
 	}
 
 	if _, found := r.tasks[t.Id]; found {
-		return fmt.Errorf("task with id `%s` already prepared", t.Id)
+		return "", fmt.Errorf("task with id `%s` already prepared", t.Id)
 	}
 
 	runnable, err := NewComposeRun(fmt.Sprintf("%s/%s", r.servicesDir, t.Service))
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	err = runnable.Prepare(env,
 		r.volumes.Path(t.Service, t.Project, t.Branch, t.Id.String()),
 		t.Id,
 		r.hosts)
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	r.tasks[t.Id] = &Context{
@@ -77,7 +77,7 @@ func (r *Runner) Prepare(t *task.Task, env map[string]string) error {
 		run:    runnable,
 	}
 
-	return nil
+	return runnable.run, nil
 }
 
 func (r *Runner) Run(t *task.Task) (int, error) {

task/task.go

@@ -1,11 +1,15 @@
 package task
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"io"
 	"regexp"
 	"time"
 
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
 	"github.com/google/uuid"
 )
 
@@ -30,7 +34,9 @@ func (s State) String() string {
 }
 
 type Task struct {
-	Id       uuid.UUID              `json:"id"`
+	Id uuid.UUID `json:"id"`
+	// Run is used to save the name of the main/master container for this service
+	Run      string                 `json:"run"`
 	Service  string                 `json:"service"`
 	Project  string                 `json:"project"`
 	Branch   string                 `json:"branch"`
@@ -55,3 +61,19 @@ func (t *Task) Validate() error {
 	}
 	return nil
 }
+
+// Logs steam logs of the current run
+func (t *Task) Logs(ctx context.Context, follow bool) (io.ReadCloser, error) {
+	mainName := fmt.Sprintf("%s_%s_%v", t.Service, t.Run, t.Id)
+	docker, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		return nil, err
+	}
+
+	return docker.ContainerLogs(ctx, mainName, types.ContainerLogsOptions{
+		ShowStdout: true,
+		ShowStderr: true,
+		Timestamps: true,
+		Follow:     follow,
+	})
+}