Merge pull request #344 from ARGOeu/devel

Version 1.0.8

Jenkinsfile

@@ -1,7 +1,7 @@
 pipeline {
     agent {
         docker {
-            image 'argo.registry:5000/epel-7-mgo1.14'
+            image 'argo.registry:5000/epel-7-mgo1.15'
             args '-u jenkins:jenkins'
         }
     }
@@ -27,7 +27,16 @@ pipeline {
                 ln -sf ${WORKSPACE}/${PROJECT_DIR} ${WORKSPACE}/go/src/github.com/ARGOeu/${PROJECT_DIR}
                 rm -rf ${WORKSPACE}/go/src/github.com/ARGOeu/${PROJECT_DIR}/${PROJECT_DIR}
                 cd ${WORKSPACE}/go/src/github.com/ARGOeu/${PROJECT_DIR}
-                go build
+                export CGO_CFLAGS"=-O2 -fstack-protector --param=ssp-buffer-size=4 -D_FORTIFY_SOURCE=2"
+                go build -buildmode=pie -ldflags "-s -w -linkmode=external -extldflags '-z relro -z now'"
+                """
+            }
+        }
+        stage('Security Tests') {
+            steps {
+                sh """
+                cd ${WORKSPACE}/go/src/github.com/ARGOeu/${PROJECT_DIR}
+                /home/jenkins/checksec.py -b ./argo-messaging
                 """
             }
         }
@@ -91,7 +100,6 @@ pipeline {
         success {
             script{
                 if ( env.BRANCH_NAME == 'devel' ) {
-                    build job: '/ARGO-utils/argo-swagger-docs', propagate: false
                     build job: '/ARGO/argodoc/devel', propagate: false
                 } else if ( env.BRANCH_NAME == 'master' ) {
                     build job: '/ARGO/argodoc/master', propagate: false

Makefile

@@ -18,7 +18,8 @@ go-build-linux-static:
 	mkdir -p ${GOPATH}/src/github.com/ARGOeu/argo-messaging
 	cp -R . ${GOPATH}/src/github.com/ARGOeu/argo-messaging
 	cd ${GOPATH}/src/github.com/ARGOeu/argo-messaging && \
-	CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o ${APPDIR}/argo-messaging-linux-static . &&\
+    export CGO_CFLAGS"=-O2 -fstack-protector --param=ssp-buffer-size=4 -D_FORTIFY_SOURCE=2"
+	GOOS=linux go build -buildmode=pie -ldflags "-s -w -linkmode=external -extldflags '-z relro -z now'" -a -installsuffix cgo -o ${APPDIR}/argo-messaging-linux-static . &&\
 	chown ${hostUID} ${APPDIR}/argo-messaging-linux-static
 
 go-test:

README.md

@@ -1,123 +1,112 @@
-[![Build Status](https://travis-ci.org/ARGOeu/argo-messaging.svg?branch=devel)](https://travis-ci.org/ARGOeu/argo-messaging)
 # ARGO Messaging
 
-> ## :warning: Warning :warning:
-> These installation instructions are meant for running the service for demo purposes. If you want to operate the service for anything else other than a simple demo, please implement a deployment model that meets your requirements.
-
-In order to build, test and run the service, recent versions of the docker-engine (>=1.12) and the docker-compose (>= 1.8.0) are required. Step 1 refers to the docker installation on Ubuntu 16.04.1, please adopt accordingly your Linux distribution or OS.
-
-## Install docker from dockerproject.org (Ubuntu 16.04.1)
-
-```shell
-$ sudo apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
-$ echo "deb https://apt.dockerproject.org/repo ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list
-$ sudo apt-get update
-$ sudo apt-cache policy docker-engine
-$ sudo apt-get install linux-image-extra-$(uname -r) linux-image-extra-virtual
-$ sudo apt-get install docker-engine
+## Description
+The ARGO Messaging Service is a Publish/Subscribe Service,
+which implements the Google PubSub protocol. 
+Instead of focusing on a single Messaging API specification 
+for handling the logic of publishing/subscribing 
+to the broker network the API focuses 
+on creating nodes of Publishers and Subscribers as a Service.
+It provides an HTTP API that enables Users/Systems to implement
+message oriented services using the Publish/Subscribe Model over plain HTTP.
+In the Publish/Subscribe paradigm, Publishers are users/systems 
+that can send messages to
+named-channels called Topics. Subscribers are users/systems that
+create Subscriptions to
+specific topics and receive messages.
+
+## Prerequisites 
+
+#### Build Requirements
+
+ - Golang 1.15
+
+#### Datastore Requirements
+  - The service has been tested with mongodb from version `3.2.22` up to `4.2.3`.
+
+#### Broker requirements
+
+  - Kafka 2.2.1
+  - Zookeeper 3.4.5
+
+#### Push Server
+In order to support push enabled subscriptions AMS relies on an external service
+that handles the actual pushing of messages, while AMS holds the configuration
+for the subscriptions.You can create push enabled subscriptions even
+when the push-server isn't available, they will be picked up automatically
+when the push-server is up and running.
+- [Push server](https://github.com/ARGOeu/ams-push-server)
+
+
+## Configuration
+
+#### Configuration Location
+Configuration for the service takes place inside a `config.json` file, that
+resides in two possible locations:
+
+1) Same folder as the binary
+
+2) `/etc/argo-messaging/config.json`
+
+#### Configuration values
+
+- `port` - port the service will bind to
+- `zookeeper_hosts` - list of zookeeper hosts, e.g. [zoo1:2181,zoo2:2181,zoo3:2181]
+- `store_host` - store host, e.g. 'mongo1:27017,mongo2:27017,mongo3:27017'
+- `store_db` - mongo db database name
+- `certificate` - /path/to/tls/certificate
+- `certificate_key` - /path/to/cert/ley
+- `certificate_authorities_dir` - dir containing CAs
+- `log_level` - DEBUG,INFO,WARNING, ERROR or FATAL
+-  `push_enabled` - (true|false) whether or not the service will support push enabled subscriptions
+- `push_tls_enabled` - (true|false), whether or not the service will communicate over TLS with the push server
+- `push_server_host` - push1.grnet.gr
+- `push_server_port` - 443
+- `verify_push_server` - (true|false) mutual TLS for the push server
+- `push_worker_token` - token for the active push worker user
+- `log_facilities` - ["syslog", "console"]  
+- `auth_option`: (`key`|`header`|`both`), where should the service look for the access token.
+
+
+#### Build & Run the service
+
+In order to build the service, inside the AMS repo issue the command:
+```bash
+go build
 ```
-
-We advise you to follow the steps described in docker manual. For Ubuntu:
-
-- Prerequisites : https://docs.docker.com/engine/installation/linux/ubuntulinux/#prerequisites
-- Install : https://docs.docker.com/engine/installation/linux/ubuntulinux/#install
-- Add a docker group [https://docs.docker.com/engine/installation/linux/ubuntulinux/#/create-a-docker-group] .
-
-**Note:** Don't forget to login logout before running the docker as a non root user. This ensures your user is running with the correct permissions.
-
-## Install docker-compose
-
-We are using version of the Compose file format. To install the latest docker-compose, follow the guidelines here: https://github.com/docker/compose/releases
-
-## Clone the argo-messaging repository
-
-```shell
-$ git clone https://github.com/ARGOeu/argo-messaging
+In order to run the service,
+```bash
+./argo-messaging
 ```
 
-## Get certificates (skip this step if you already have certificates)
+## X509 Authentication
+Although AMS doesn't support direct authentication through an x509 certificate,
+you can use the [argo-authentication-service](https://github.com/ARGOeu/argo-api-authn)
+to map an x509 certificate to an AMS `key`.
+The service will also validate the certificate.
+The [ams-library](https://github.com/ARGOeu/argo-ams-library) will effortlessly
+hide this complexity if you decide to use it in order to access AMS.
 
-The ARGO Messaging services requires certificates in order to operates. The easiest way is to get certificates from letsencrypt. You can follow the instructions from the letsencrypt website or use the docker letsencrypt docker image. One caveat of this approach is that the certificate files end up in the ```etc/live``` directory (see below) and will be owned by the root user.
+## Managing the protocol buffers and gRPC definitions
 
-```shell
-$ mkdir -p ${HOME}/letsencrypt/{etc,var}
-$ docker run -it --rm -p 443:443 -p 80:80 --name certbot \
-    -v "$HOME/letsencrypt/etc:/etc/letsencrypt" \
-    -v "$HOME/letsencrypt/var:/var/lib/letsencrypt" \
-    quay.io/letsencrypt/letsencrypt:latest certonly
-$ cd argo-messaging
-# Comment: Please change owneship of ${HOME}/letsencrypt to your user
-$ cp ${HOME}/letsencrypt/etc/live/*/fullchain.pem host.crt
-$ sudo cp ${HOME}/letsencrypt/etc/live/*/privkey.pem host.key
-```
-## Edit the default configuration file (config.json)
-
-In the ```argo-messaging``` directory, edit ```config.json```:
-
-```diff
-{
-"bind_ip":"",
-"port":8080,
--  "zookeeper_hosts":["localhost"],
--  "store_host":"localhost",
-+  "zookeeper_hosts":["zookeeper"],
-+  "store_host":"mongo",
-"store_db":"argo_msg",
--  "certificate":"/etc/pki/tls/certs/localhost.crt",
--  "certificate_key":"/etc/pki/tls/private/localhost.key",
-+  "certificate":"./host.crt",
-+  "certificate_key":"./host.key",
-"service_token":"CHANGE-THIS-TO-A-LONG-STRING",
-"push_enabled": false
-}
-```
-
-**Note:** Make sure that you change the service_token to a long string.
+In order to modify any `.proto` file you will need the following
 
-## Edit docker-compose.yml
+ - Read on how to install the protoc compiler on your platform [here.](https://github.com/protocolbuffers/protobuf)
 
-In the ```argo-messaging``` directory, edit ```docker-compose.yml``` and add the public IP address of your host to the ```KAFKA_ADVERTISED_HOST_NAME``` key.
+ -  Install the go plugin. `go get -u github.com/golang/protobuf/protoc-gen-go`
 
-## Run the tests
+ - install the go gRPC package. `go get -u google.golang.org/grpc`
 
-```shell
-$ docker run --env hostUID=`id -u`:`id -g` --rm -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.7 make go-test
-```
+ - Inside `push/grpc` compile. `protoc -I proto/ proto/ams.proto --go_out=plugins=grpc:proto`
 
-## Build the service
+## Helpful utilities
 
-```shell
-$ docker run --env hostUID=`id -u`:`id -g` --rm -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.7 make go-build-linux-static
-```
-
-## Start the service
-
-```shell
-$ docker-compose build
-$ docker-compose up -d
-```
-
-##  Test that the service is running
-
-```shell
-$ curl https://<HOSTNAME>/v1/projects?key=<YOUR_SERVICE_TOKEN>
-```
-
-**Note:** Change ```<HOSTNAME>``` to the hostname of your host and ```<SERVICE_TOKEN>``` to the service token that you have added in ```config.json```. You should get an empty json response:
-
-```shell
-{}
-```
-
-## Stop the service
-
-```shell
-$ docker-compose stop
-```
+Inside the [tools](https://github.com/ARGOeu/argo-messaging/tree/master/tools) folder you can find various scripts that can help you
+perform common tasks OR help you get started with interacting with AMS.
 
-## Congratulations!
+There is also a handy python [library]((https://github.com/ARGOeu/argo-ams-library))
+for interacting with AMS.
 
-Please visit http://argoeu.github.io/messaging/v1/ to learn how to use the service.
 
 ## Credits
 

argo-messaging.spec

@@ -3,13 +3,12 @@
 
 Name: argo-messaging
 Summary: ARGO Messaging API for broker network
-Version: 1.0.7
+Version: 1.0.8
 Release: 1%{?dist}
 License: ASL 2.0
 Buildroot: %{_tmppath}/%{name}-buildroot
 Group: Unspecified
 Source0: %{name}-%{version}.tar.gz
-BuildRequires: golang
 BuildRequires: git
 Requires(pre): /usr/sbin/useradd, /usr/bin/getent
 ExcludeArch: i386
@@ -31,7 +30,8 @@ export PATH=$PATH:$GOPATH/bin
 cd src/github.com/ARGOeu/argo-messaging/
 export GIT_COMMIT=$(git rev-list -1 HEAD)
 export BUILD_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-go install -ldflags "-X github.com/ARGOeu/argo-messaging/version.Commit=$GIT_COMMIT -X github.com/ARGOeu/argo-messaging/version.BuildTime=$BUILD_TIME"
+export CGO_CFLAGS"=-O2 -fstack-protector --param=ssp-buffer-size=4 -D_FORTIFY_SOURCE=2"
+go install -buildmode=pie -ldflags "-s -w -linkmode=external -extldflags '-z relro -z now' -X github.com/ARGOeu/argo-messaging/version.Commit=$GIT_COMMIT -X github.com/ARGOeu/argo-messaging/version.BuildTime=$BUILD_TIME"
 
 %install
 %{__rm} -rf %{buildroot}
@@ -63,6 +63,8 @@ go clean
 %attr(0644,root,root) /usr/lib/systemd/system/argo-messaging.service
 
 %changelog
+* Wed Mar 31 2021 Agelos Tsalapatis  <[email protected]> 1.0.8-1%{?dist}
+- AMS release 1.0.8
 * Wed Aug 05 2020 Agelos Tsalapatis  <[email protected]> 1.0.7-1%{?dist}
 - AMS release 1.0.7
 * Wed Jan 08 2020 Agelos Tsalapatis  <[email protected]> 1.0.6-1%{?dist}