diff options
| -rw-r--r-- | Dockerfile | 10 | ||||
| -rw-r--r-- | conf/nginx.conf | 5 | ||||
| -rw-r--r-- | conf/supervisord.conf | 6 | ||||
| -rw-r--r-- | omaha_server/omaha_server/settings_prod.py | 19 | ||||
| -rw-r--r-- | omaha_server/omaha_server/utils.py | 13 | ||||
| -rw-r--r-- | pavement.py | 24 | ||||
| -rw-r--r-- | requirements/base.txt | 1 |
7 files changed, 40 insertions, 38 deletions
@@ -3,6 +3,8 @@ FROM ubuntu-debootstrap:14.04 ENV omaha /srv/omaha RUN \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys ABF5BD827BD9BF62 && \ + echo 'deb http://nginx.org/packages/ubuntu/ trusty nginx' | tee --append /etc/apt/sources.list && \ apt-get update && \ apt-get install -y --no-install-recommends python-pip python-lxml python-psycopg2 uwsgi supervisor uwsgi-plugin-python nginx libtiff5-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev python-pil build-essential libfuse-dev libcurl4-openssl-dev libxml2-dev mime-support automake libtool pkg-config libssl-dev wget tar && \ apt-get clean && \ @@ -35,16 +37,12 @@ ADD . $omaha # setup all the configfiles RUN \ - rm /etc/nginx/sites-enabled/default && \ + mkdir /etc/nginx/sites-enabled/ && \ + rm /etc/nginx/conf.d/default.conf && \ rm /etc/nginx/nginx.conf && \ ln -s /srv/omaha/conf/nginx.conf /etc/nginx/ && \ ln -s /srv/omaha/conf/nginx-app.conf /etc/nginx/sites-enabled/ && \ ln -s /srv/omaha/conf/supervisord.conf /etc/supervisor/conf.d/ -RUN \ - wget -O /tmp/splunkforwarder-6.3.1-f3e41e4b37b2-linux-2.6-amd64.deb 'http://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version=6.3.1&product=universalforwarder&filename=splunkforwarder-6.3.1-f3e41e4b37b2-linux-2.6-amd64.deb&wget=true' && \ - dpkg -i /tmp/splunkforwarder-6.3.1-f3e41e4b37b2-linux-2.6-amd64.deb && \ - rm /tmp/splunkforwarder-6.3.1-f3e41e4b37b2-linux-2.6-amd64.deb - EXPOSE 80 CMD ["paver", "docker_run"] diff --git a/conf/nginx.conf b/conf/nginx.conf index b1faf6d..2f898d7 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -43,7 +43,10 @@ http { # Logging Settings ## - access_log /var/log/nginx/access.log; + log_format main '$remote_addr $remote_user [$time_local] hostname=$server_name $status ' + '"$request" $body_bytes_sent "$http_referer" ' + '"$http_user_agent" '; + access_log /var/log/nginx/access.log main; error_log /var/log/nginx/error.log; ## diff --git a/conf/supervisord.conf b/conf/supervisord.conf index 117403a..87f37b5 100644 --- a/conf/supervisord.conf +++ b/conf/supervisord.conf @@ -22,13 +22,7 @@ startsecs=10 stopwaitsecs = 600 killasgroup=true - [program:s3fs] command=/usr/bin/s3fs %(ENV_AWS_STORAGE_BUCKET_NAME)s /srv/omaha_s3 -f -ouse_cache=/tmp -oiam_role=%(ENV_AWS_ROLE)s autostart=true autorestart=true - -[program:splunk-forwarder] -command=/opt/splunkforwarder/bin/splunk start -autostart=true -autorestart=true
\ No newline at end of file diff --git a/omaha_server/omaha_server/settings_prod.py b/omaha_server/omaha_server/settings_prod.py index a993adb..7695043 100644 --- a/omaha_server/omaha_server/settings_prod.py +++ b/omaha_server/omaha_server/settings_prod.py @@ -33,6 +33,7 @@ RAVEN_DSN_STACKTRACE = os.environ.get('RAVEN_DSN_STACKTRACE', RAVEN_CONFIG['dsn' SPLUNK_HOST = os.environ.get('SPLUNK_HOST') +SPLUNK_PORT = os.environ.get('SPLUNK_PORT', None) INSTALLED_APPS = INSTALLED_APPS + ( 'raven.contrib.django.raven_compat', @@ -42,7 +43,7 @@ CELERYD_HIJACK_ROOT_LOGGER = False LOGGING = { 'version': 1, - 'disable_existing_loggers': False, + 'disable_existing_loggers': True, 'root': { 'level': 'INFO', 'handlers': ['sentry', 'console'], @@ -52,7 +53,7 @@ LOGGING = { 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' }, 'splunk_format':{ - 'format': 'level=%(levelname)s logger=%(name)s timestamp=%(asctime)s module=%(module)s process=%(process)d thread=%(thread)d message=%(message)s\n\r' + 'format': 'hostname={} level=%(levelname)s logger=%(name)s timestamp=%(asctime)s module=%(module)s process=%(process)d thread=%(thread)d message=%(message)s\n\r'.format(HOST_NAME) } }, 'handlers': { @@ -85,19 +86,11 @@ LOGGING = { }, } -if SPLUNK_HOST: +if SPLUNK_HOST and SPLUNK_PORT: LOGGING['handlers']['splunk'] = { 'level': os.environ.get('SPLUNK_LOGGING_LEVEL', 'INFO'), - 'class': 'splunk_handler.SplunkHandler', + 'class': 'omaha_server.utils.CustomSysLogHandler', 'formatter': 'splunk_format', - 'host': SPLUNK_HOST, - 'port': os.environ.get('SPLUNK_MANAGEMENT_PORT', 8089), - 'username': os.environ.get('SPLUNK_USERNAME', 'admin'), - 'password': os.environ.get('SPLUNK_PASSWORD', 'changeme'), - 'hostname': HOST_NAME or 'Unknown', - 'index': 'main', - 'source': 'omaha', - 'sourcetype': 'omaha-server', - 'verify': False, + 'address': (SPLUNK_HOST, int(SPLUNK_PORT)) } LOGGING['root']['handlers'].append('splunk') diff --git a/omaha_server/omaha_server/utils.py b/omaha_server/omaha_server/utils.py index ab6f851..d10934c 100644 --- a/omaha_server/omaha_server/utils.py +++ b/omaha_server/omaha_server/utils.py @@ -1,6 +1,7 @@ # coding: utf8 from functools import wraps +import logging.handlers from django.conf import settings @@ -52,3 +53,15 @@ def get_splunk_url(params): splunk_host = getattr(settings, 'SPLUNK_HOST', None) string_params = ' '.join("%s=%s" % (key, val) for (key, val) in sorted(params.items())) return SEARCH_TEMPLATE % (splunk_host, string_params) if splunk_host else None + +class CustomSysLogHandler(logging.handlers.SysLogHandler): + def emit(self, record): + msg = self.format(record) + '\000' + if type(msg) is unicode: + msg = msg.encode('utf-8') + try: + self.socket.sendto(msg, self.address) + except (KeyboardInterrupt, SystemExit): + raise + except: + self.handleError(record) diff --git a/pavement.py b/pavement.py index 478f184..cfbb606 100644 --- a/pavement.py +++ b/pavement.py @@ -90,18 +90,20 @@ def migrate(): def create_admin(): sh('./createadmin.py', cwd='omaha_server') + @task -def configure_splunk_forwarder(): - hostname = os.environ.get('HOST_NAME') +def configure_nginx(): splunk_host = os.environ.get('SPLUNK_HOST') - splunk_receiving_port = os.environ.get('SPLUNK_RECEIVING_PORT', 9997) - if splunk_host: - try: - sh('/opt/splunkforwarder/bin/splunk add forward-server %s:%s --accept-license -auth admin:changeme' % (splunk_host, splunk_receiving_port)) - sh('/opt/splunkforwarder/bin/splunk add monitor /var/log/nginx -index main -sourcetype Nginx') - sh('echo "[default] \nhost = %s \n" > /opt/splunkforwarder/etc/system/local/inputs.conf' % hostname) - except: - pass + splunk_port = os.environ.get('SPLUNK_PORT', '') + if splunk_host and splunk_port.isdigit(): + sh("sed -i 's/access_log.*;/access_log syslog:server=%s:%s main;/g' /etc/nginx/nginx.conf" % (splunk_host, splunk_port)) + sh("sed -i 's/error_log.*;/error_log syslog:server=%s:%s;/g' /etc/nginx/nginx.conf" % (splunk_host, splunk_port)) + else: + sh("sed -i 's#access_log.*;#access_log /var/log/nginx/access.log main;#g' /etc/nginx/nginx.conf") + sh("sed -i 's#error_log.*;#error_log /var/log/nginx/error.log;#g' /etc/nginx/nginx.conf") + server_name = os.environ.get('HOST_NAME', '_') + sh("sed -i 's/server_name.*;/server_name %s;/g' /etc/nginx/sites-enabled/nginx-app.conf" % (server_name)) + @task def docker_run(): @@ -114,7 +116,7 @@ def docker_run(): create_admin() collectstatic() - configure_splunk_forwarder() + configure_nginx() sh('/usr/bin/supervisord') except: client.captureException() diff --git a/requirements/base.txt b/requirements/base.txt index 785ba1a..90d9ddb 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -31,7 +31,6 @@ django-bootstrap3==6.2.2 protobuf==3.0.0a3 protobuf-to-dict==0.1.0 django-dynamic-preferences==0.6.1 -splunk-handler==1.1.3 # Only dev #django-httplog==0.2.3 |