Merge pull request #14 from totem/develop

0.2 Release:  Centralized Logging Support

Author: sukrit007

Dockerfile

@@ -3,15 +3,11 @@ ENV DEBIAN_FRONTEND noninteractive
 ENV ETCDCTL_VERSION v0.4.6
 
 RUN apt-get update && \
-    apt-get install -y haproxy/trusty-backports openssh-server nano
+    apt-get install -y haproxy/trusty-backports nano
 
 #AWS Cli and Supervisor
 RUN pip install awscli==1.4.1 supervisor==3.1.2
 
-#Syslog
-RUN echo '$PreserveFQDN on' | cat - /etc/rsyslog.conf > /tmp/rsyslog.conf && sudo mv /tmp/rsyslog.conf /etc/rsyslog.conf && \
-    sed -i 's~^#\$ModLoad immark\(.*\)$~$ModLoad immark \1~' /etc/rsyslog.conf
-
 #Haproxy
 RUN mkdir -p /run/haproxy && chown -R haproxy:haproxy /run/haproxy
 ADD ./bin/haproxy-wrapper.sh /usr/sbin/haproxy-wrapper.sh
@@ -23,13 +19,6 @@ RUN curl -L https://github.com/coreos/etcd/releases/download/$ETCDCTL_VERSION/et
     cp -f /tmp/etcd-$ETCDCTL_VERSION-linux-amd64/etcdctl /usr/local/bin && \
     rm -rf /tmp/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz
 
-##SSH Server (To troubleshoot issues with image factory)
-RUN mkdir /var/run/sshd
-ADD root/.ssh/authorized_keys /root/.ssh/authorized_keys
-RUN chmod -R 400 /root/.ssh/* && \
-    chmod  500 /root/.ssh && \
-    chown -R root:root /root/.ssh
-
 #Supervisor
 #Supervisor Config
 RUN mkdir -p /var/log/supervisor && \
@@ -39,7 +28,7 @@ ADD etc/supervisor/supervisord.conf /etc/supervisor/
 RUN ln -sf /etc/supervisor/supervisord.conf /etc/supervisord.conf
 
 #Confd
-ENV CONFD_VERSION 0.6.3
+ENV CONFD_VERSION 0.7.1
 RUN curl -L https://github.com/kelseyhightower/confd/releases/download/v$CONFD_VERSION/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd && \
     chmod 555 /usr/local/bin/confd
 
@@ -61,8 +50,9 @@ ENV ETCD_PROXY_BASE /yoda
 ENV PROXY_HOST yoda.local.sh
 ENV SYNC_CERTS false
 ENV S3_YODA_BUCKET yoda-certs
+ENV LOG_IDENTIFIER yoda-proxy
 
-EXPOSE 22 80 443 8081
+EXPOSE 80 443 8081
 
 ENTRYPOINT ["/usr/local/bin/supervisord"]
 CMD ["-n"]

README.md

@@ -47,7 +47,7 @@ Private Key, Public Key, CA Chain (In this order).
 Once ssl bucket is setup, simply run command below to start your proxy:  
 
 ```
-sudo docker run --name yoda --rm -t -i -P -p 80:80 -p 443:443 -p 2022:22 -e AWS_ACCESS_KEY_ID=<<S3_ACCESS_KEY_ID>> -e AWS_SECRET_ACCESS_KEY=<<S3_SECRET_KEY>> -e S3_YODA_BUCKET=<<S3_BUCKET_NAME>> -e SYNC_CERTS=true totem/yoda-proxy
+sudo docker run --name yoda --rm -t -i -v /dev/log:/dev/log -P -p 80:80 -p 443:443 -p 2022:22 -e AWS_ACCESS_KEY_ID=<<S3_ACCESS_KEY_ID>> -e AWS_SECRET_ACCESS_KEY=<<S3_SECRET_KEY>> -e S3_YODA_BUCKET=<<S3_BUCKET_NAME>> -e SYNC_CERTS=true totem/yoda-proxy
 ```
 
 ## ETCD Configuration

bin/confd-wrapper.sh

@@ -21,10 +21,6 @@ $ETCDCTL get $LISTENER_PATH/admin/bind || $ETCDCTL set $LISTENER_PATH/admin/bind
 $ETCDCTL get $ETCD_PROXY_BASE/global/acls/public/cidr/src || $ETCDCTL set $ETCD_PROXY_BASE/global/acls/public/cidr/src "0.0.0.0/0"
 $ETCDCTL get $ETCD_PROXY_BASE/global/acls/global-black-list/cidr/src || $ETCDCTL set $ETCD_PROXY_BASE/global/acls/global-black-list/cidr/src ""
 
-#Syslog ETCD Entries
-$ETCDCTL get $ETCD_PROXY_BASE/syslog/host || $ETCDCTL set $ETCD_PROXY_BASE/syslog/host ""
-
-
-sed -i -e "s/172.17.42.1[:]4001/$ETCD_URL/g" -e "s|/yoda|$ETCD_PROXY_BASE|g" /etc/confd/confd.toml
+sed -i -e "s/172.17.42.1[:]4001/$ETCD_URL/g" -e "s|/yoda|$ETCD_PROXY_BASE|" /etc/confd/confd.toml
 #sed -i -e "s|/yoda|$ETCD_PROXY_BASE|g" /etc/confd/conf.d/haproxy.toml
 confd

etc/confd/conf.d/10-syslog-central.toml

@@ -1,6 +1,6 @@
 {{/* Template for CIDRS: Required for reuse in Http and Https frontends */}}
 {{define "CIDR_ACLS"}}{{if ls "/global/acls"}}{{ range $acl := gets "/global/acls/*/cidr/src" }}
-    acl {{$acl.Key | parent | parent | base}} src {{$acl.Value}}
+    acl {{$acl.Key | dir | dir | base}} src {{$acl.Value}}
 {{ end }}{{ end }}{{ end }}
 
 {{/* Global Black Listing */}}
@@ -14,14 +14,14 @@
 {{ end }}{{ end }}
 
 {{/* Template for use_backends. */}}
-{{define "USE_BACKENDS"}}{{ range $host := ls "/hosts" }}{{if printf "/hosts/%s/locations" $host | ls }}{{ range $path := printf "/hosts/%s/locations/*/path" $host | gets}}{{ $pathName := $path.Key | parent | base }}
+{{define "USE_BACKENDS"}}{{ range $host := ls "/hosts" }}{{if printf "/hosts/%s/locations" $host | ls }}{{ range $path := printf "/hosts/%s/locations/*/path" $host | gets}}{{ $pathName := $path.Key | dir | base }}
 {{/* Do not include backends if upstream is empty or if it does not exist */}}
     {{ $upstream := printf "/hosts/%s/locations/%s/upstream" $host $pathName | getv}}{{ $endpoints := printf "/upstreams/%s/endpoints" $upstream | ls}}
     {{if and $upstream $endpoints}}use_backend {{printf "/hosts/%s/locations/%s/upstream" $host $pathName | getv }} if __host-{{$host}} __path-{{$host}}-{{$pathName}}{{ end }}
 {{ end }}{{ end }}{{ end }}{{ end }}
 
 {{/* Template for force ssl */}}
-{{define "FORCE_SSL"}}{{ range $host := ls "/hosts" }}{{if printf "/hosts/%s/locations" $host | ls }}{{ range $path := printf "/hosts/%s/locations/*/path" $host | gets}}{{ $pathName := $path.Key | parent | base }}
+{{define "FORCE_SSL"}}{{ range $host := ls "/hosts" }}{{if printf "/hosts/%s/locations" $host | ls }}{{ range $path := printf "/hosts/%s/locations/*/path" $host | gets}}{{ $pathName := $path.Key | dir | base }}
     {{ if printf "/hosts/%s/locations/%s" $host $pathName | ls}}{{ if printf "/hosts/%s/locations/%s/force-ssl" $host $pathName | ls}}{{ if printf "/hosts/%s/locations/%s/force-ssl" $host $pathName | getv | eq "true"}}
     #Force SSL for $host/$pathName
     redirect scheme https code 301 if !{ ssl_fc } __host-{{$host}} __path-{{$host}}-{{$pathName}}
@@ -33,7 +33,7 @@
     {{ range $host := ls "/hosts" }}
     #Host ACL for host:{{$host}}
     acl __host-{{$host}} hdr_dom(host) -i -m str {{$host}} {{ if printf "/hosts/%s/aliases" $host | ls }}{{range $alias := printf "/hosts/%s/aliases/*" $host | getvs }}{{$alias}} {{ end }}{{ end }}
-    {{ range $path := printf "/hosts/%s/locations/*/path" $host | gets}}{{ $pathName := $path.Key | parent | base }}
+    {{ range $path := printf "/hosts/%s/locations/*/path" $host | gets}}{{ $pathName := $path.Key | dir | base }}
     #Path ACL for host:{{$host}} and path:{{$path}}
     acl __path-{{$host}}-{{$pathName}} path_beg {{$path.Value}}
 {{/* Deny Requests if host, path matches and any of cidrs matches */}}
@@ -56,6 +56,7 @@
 global
     log /dev/log	local0
     log /dev/log	local1 notice
+    log-tag {{getenv "LOG_IDENTIFIER"}}
     chroot /var/lib/haproxy
     stats socket /run/haproxy/admin.sock mode 660 level admin
     stats timeout 30s
@@ -184,7 +185,7 @@ backend {{ $upstream }}
    {{ $health_interval := printf "%s/interval" $health }}
    {{ if $health_uri | ls }}option httpchk GET {{ $health_uri | getv }}{{ end }}
    {{ if $health_timeout | ls }}timeout check {{ $health_timeout | getv }}{{ end }}
-   {{ range $endpoint := printf "/upstreams/%s/endpoints/*" $upstream | gets }}{{ $endpointName := $endpoint.Key | parent | base }}
+   {{ range $endpoint := printf "/upstreams/%s/endpoints/*" $upstream | gets }}{{ $endpointName := $endpoint.Key | dir | base }}
    server {{ $endpointName }} {{ $endpoint.Value }}  check inter {{ if $health_interval | ls }}{{$health_interval | getv}}{{ else }}2m{{ end }}
    {{ end }}
 {{ end }}{{ end }}

etc/confd/templates/10-syslog-central.conf.tmpl

@@ -5,32 +5,19 @@ nodaemon=true
 command=/usr/sbin/haproxy-wrapper.sh
 autorestart=true
 startsecs=5
-redirect_stderr=true
+stderr_logfile=syslog
 stdout_logfile=syslog
 
 [program:confd]
 command=/usr/sbin/confd-wrapper.sh
 autorestart=true
 startsecs=5
-redirect_stderr=true
-stdout_logfile=syslog
-
-[program:sshd]
-command=/usr/sbin/sshd -D
-autorestart=true
-startsecs=5
-redirect_stderr=true
+stderr_logfile=syslog
 stdout_logfile=syslog
 
 [program:synccerts]
 command=/usr/sbin/sync-certs.sh
 autorestart=true
 startsecs=5
-redirect_stderr=true
-stdout_logfile=syslog
-
-[program:rsyslog]
-command=rsyslogd -n -c5
-autostart=true
-autorestart=true
-redirect_stderr=true
+stderr_logfile=syslog
+stdout_logfile=syslog

etc/confd/templates/haproxy.cfg.tmpl

@@ -11,7 +11,7 @@
 from threading import Thread
 
 DOCKER = os.environ.get('DOCKER_CMD', 'docker -H 127.0.0.1:8283')
-ETCD_PROXY_BASE = os.environ.get('ETCD_PROXY_BASE', '/yoda-integration')
+ETCD_PROXY_BASE = os.environ.get('ETCD_PROXY_BASE', '/test-yoda-integration')
 ETCD_HOST = os.environ.get('ETCD_HOST', 'localhost')
 ETCD_PORT = int(os.environ.get('ETCD_PORT', '4001'))
 MOCK_TCP_PORT = int(os.environ.get('MOCK_TCP_PORT', '31325'))
@@ -33,6 +33,7 @@ def setup_yoda():
     build_yoda()
     check_call(
         '{DOCKER} run --name yoda-integration -d -P -p 80:80 -p 443:443 '
+        '-v /dev/log:/dev/log '
         '-p {MOCK_TCP_PORT}:{MOCK_TCP_PORT} -e ETCD_PROXY_BASE'
         '={ETCD_PROXY_BASE} -h yoda-integration-{USER} totem/yoda-integration'
         .format(DOCKER=DOCKER, ETCD_PROXY_BASE=ETCD_PROXY_BASE,