task processing_crash_dump & unittest

14 files changed, 178 insertions, 5 deletions

diff --git a/Dockerfile b/Dockerfile
index a3aab08..5fdf06f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,6 +24,7 @@ RUN apt-get clean
 RUN wget https://github.com/s3fs-fuse/s3fs-fuse/archive/v1.78.tar.gz -O /usr/src/v1.78.tar.gz
 RUN tar xvz -C /usr/src -f /usr/src/v1.78.tar.gz
 RUN cd /usr/src/s3fs-fuse-1.78 && ./autogen.sh && ./configure --prefix=/usr && make && make install
+RUN mkdir /srv/omaha_s3
 
 
 ADD . /srv/omaha
diff --git a/conf/supervisord.conf b/conf/supervisord.conf
index bd9e43e..d7c4d51 100644
--- a/conf/supervisord.conf
+++ b/conf/supervisord.conf
@@ -10,7 +10,7 @@ command=uwsgi --ini /srv/omaha/conf/uwsgi.ini
 [program:celery]
 command=celery worker -A omaha_server --loglevel=INFO
 directory=/srv/omaha/omaha_server
-user=nobody
+environment=C_FORCE_ROOT="true"
 numprocs=1
 autostart=true
 autorestart=true
diff --git a/omaha_server/crash/migrations/0004_crash_stacktrace.py b/omaha_server/crash/migrations/0004_crash_stacktrace.py
new file mode 100644
index 0000000..c517b7b
--- /dev/null
+++ b/omaha_server/crash/migrations/0004_crash_stacktrace.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('crash', '0003_auto_20141209_1153'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='crash',
+            name='stacktrace',
+            field=models.TextField(null=True, blank=True),
+            preserve_default=True,
+        ),
+    ]
diff --git a/omaha_server/crash/models.py b/omaha_server/crash/models.py
index ba17392..0fc8f5e 100644
--- a/omaha_server/crash/models.py
+++ b/omaha_server/crash/models.py
@@ -21,7 +21,10 @@ the License.
 import os
 
 from django.db import models
+from django.db.models.signals import post_save
+from django.dispatch import receiver
 
+from celery import signature
 from django_extensions.db.models import TimeStampedModel
 from jsonfield import JSONField
 
@@ -33,6 +36,7 @@ class Crash(TimeStampedModel):
     app_id = models.CharField(max_length=38, null=True, blank=True)
     user_id = models.CharField(max_length=38, null=True, blank=True)
     meta = JSONField(verbose_name='Meta-information', help_text='JSON format', null=True, blank=True)
+    stacktrace = models.TextField(null=True, blank=True)
 
 
 def symbols_upload_to(obj, filename):
@@ -49,3 +53,9 @@ class Symbols(TimeStampedModel):
 
     class Meta:
         verbose_name_plural = 'Symbols'
+
+
+@receiver(post_save, sender=Crash)
+def crash_post_save(sender, instance, created, *args, **kwargs):
+    if created:
+        signature("tasks.processing_crash_dump", args=(instance.pk,)).apply_async(queue='default')
diff --git a/omaha_server/crash/settings.py b/omaha_server/crash/settings.py
index 7da1134..5df6d00 100644
--- a/omaha_server/crash/settings.py
+++ b/omaha_server/crash/settings.py
@@ -35,3 +35,4 @@ MINIDUMP_STACKWALK_PATH = os.path.join(BASE_DIR, 'breakpad/%s/minidump_stackwalk
 
 MINIDUMP_STACKWALK_PATH = getattr(settings, 'CRASH_MINIDUMP_STACKWALK_PATH', MINIDUMP_STACKWALK_PATH)
 SYMBOLS_PATH = getattr(settings, 'CRASH_SYMBOLS_PATH')
+S3_MOUNT_PATH = getattr(settings, 'CRASH_S3_MOUNT_PATH')
diff --git a/omaha_server/crash/tasks.py b/omaha_server/crash/tasks.py
new file mode 100644
index 0000000..07eedad
--- /dev/null
+++ b/omaha_server/crash/tasks.py
@@ -0,0 +1,45 @@
+# coding: utf8
+
+"""
+This software is licensed under the Apache 2 license, quoted below.
+
+Copyright 2014 Crystalnix Limited
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not
+use this file except in compliance with the License. You may obtain a copy of
+the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations under
+the License.
+"""
+
+import os
+
+from clom import shell
+from furl import furl
+from raven.contrib.django.raven_compat.models import client
+
+from omaha_server.celery import app
+from models import Crash
+from settings import S3_MOUNT_PATH
+from utils import get_stacktrace
+
+
+@app.task(name='tasks.processing_crash_dump', ignore_result=True, max_retries=12)
+def processing_crash_dump(crash_pk):
+    try:
+        crash = Crash.objects.get(pk=crash_pk)
+        url = furl(crash.mini_dump.url)
+        path = url.pathstr
+        crash_dump_path = os.path.join(S3_MOUNT_PATH, *path.split('/'))
+        stacktrace, errors = get_stacktrace(crash_dump_path)
+        crash.stacktrace = stacktrace
+        crash.save()
+    except Crash.DoesNotExist, shell.CommandError:
+        client.captureException()
+        raise processing_crash_dump.retry(countdown=2 ** processing_crash_dump.request.retries)
diff --git a/omaha_server/crash/tests/test_models.py b/omaha_server/crash/tests/test_models.py
index 1cfba3c..9482d62 100644
--- a/omaha_server/crash/tests/test_models.py
+++ b/omaha_server/crash/tests/test_models.py
@@ -34,7 +34,10 @@ SYM_FILE = os.path.join(TEST_DATA_DIR, 'BreakpadTestApp.sym')
 
 
 class CrashModelTest(test.TestCase):
-    @temporary_media_root()
+    @temporary_media_root(
+        CELERY_ALWAYS_EAGER=False,
+        CELERY_EAGER_PROPAGATES_EXCEPTIONS=False,
+    )
     def test_model(self):
         meta = dict(
             lang='en',
@@ -85,5 +88,3 @@ class SymbolsModelTest(test.TestCase):
 
         self.assertEqual(symbols_upload_to(obj, 'BreakpadTestApp.pdb'),
                          'symbols/BreakpadTestApp.pdb/C1C0FA629EAA4B4D9DD2ADE270A231CC1/BreakpadTestApp.sym')
-
-
diff --git a/omaha_server/crash/tests/test_tasks.py b/omaha_server/crash/tests/test_tasks.py
new file mode 100644
index 0000000..b0a1145
--- /dev/null
+++ b/omaha_server/crash/tests/test_tasks.py
@@ -0,0 +1,70 @@
+# coding: utf8
+
+"""
+This software is licensed under the Apache 2 license, quoted below.
+
+Copyright 2014 Crystalnix Limited
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not
+use this file except in compliance with the License. You may obtain a copy of
+the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations under
+the License.
+"""
+
+import os
+
+from django import test
+from django.core.files.uploadedfile import SimpleUploadedFile
+
+from freezegun import freeze_time
+
+from crash.models import Crash
+from crash.tasks import processing_crash_dump
+from omaha.tests.utils import temporary_media_root
+
+
+BASE_DIR = os.path.dirname(__file__)
+TEST_DATA_DIR = os.path.join(BASE_DIR, 'testdata')
+SYM_FILE = os.path.join(TEST_DATA_DIR, 'BreakpadTestApp.sym')
+CRASH_DUMP_FILE = os.path.join(TEST_DATA_DIR, '7b05e196-7e23-416b-bd13-99287924e214.dmp')
+SYMBOLS_PATH = os.path.join(TEST_DATA_DIR, 'symbols')
+STACKTRACE_PATH = os.path.join(TEST_DATA_DIR, 'stacktrace.txt')
+
+
+class CrashModelTest(test.TestCase):
+    @temporary_media_root(
+        MEDIA_URL='http://omaha-test.s3.amazonaws.com/',
+        CRASH_S3_MOUNT_PATH=TEST_DATA_DIR,
+        CRASH_SYMBOLS_PATH=SYMBOLS_PATH,
+    )
+    @freeze_time("2014-12-11")
+    def test_model(self):
+        meta = dict(
+            lang='en',
+            version='1.0.0.1',
+        )
+        app_id = '{D0AB2EBC-931B-4013-9FEB-C9C4C2225C8C}',
+        user_id = '{2882CF9B-D9C2-4edb-9AAF-8ED5FCF366F7}',
+        with open(CRASH_DUMP_FILE) as f:
+            obj = Crash.objects.create(
+                app_id=app_id,
+                user_id=user_id,
+                mini_dump=SimpleUploadedFile('7b05e196-7e23-416b-bd13-99287924e214.dmp', f.read()),
+                meta=meta,
+            )
+
+        self.assertIsNone(obj.stacktrace)
+        processing_crash_dump(obj.pk)
+
+        with open(STACKTRACE_PATH, 'rb') as f:
+            stacktrace = f.read()
+
+        crash = Crash.objects.get(pk=obj.pk)
+        self.assertEqual(crash.stacktrace, stacktrace)
diff --git a/omaha_server/crash/tests/test_views.py b/omaha_server/crash/tests/test_views.py
index 7db7f5b..59d368c 100644
--- a/omaha_server/crash/tests/test_views.py
+++ b/omaha_server/crash/tests/test_views.py
@@ -26,6 +26,10 @@ from crash.models import Crash
 
 
 class CrashViewTest(test.TestCase):
+    @test.override_settings(
+        CELERY_ALWAYS_EAGER=False,
+        CELERY_EAGER_PROPAGATES_EXCEPTIONS=False,
+    )
     def test_view(self):
         meta = dict(
             lang='en',
diff --git a/omaha_server/crash/tests/testdata/minidump/2014/12/11/7b05e196-7e23-416b-bd13-99287924e214.dmp b/omaha_server/crash/tests/testdata/minidump/2014/12/11/7b05e196-7e23-416b-bd13-99287924e214.dmp
new file mode 100644
index 0000000..3a1e4e1
--- /dev/null
+++ b/omaha_server/crash/tests/testdata/minidump/2014/12/11/7b05e196-7e23-416b-bd13-99287924e214.dmp
diff --git a/omaha_server/omaha_server/settings.py b/omaha_server/omaha_server/settings.py
index 79589c9..845a73d 100644
--- a/omaha_server/omaha_server/settings.py
+++ b/omaha_server/omaha_server/settings.py
@@ -193,3 +193,8 @@ CACHEOPS_REDIS = {
 CACHEOPS = {
     'omaha.*': {'ops': (), 'timeout': 10},
 }
+
+# Crash
+
+CRASH_S3_MOUNT_PATH = os.environ.get('CRASH_S3_MOUNT_PATH', '/srv/omaha_s3')
+CRASH_SYMBOLS_PATH = os.path.join(CRASH_S3_MOUNT_PATH, 'symbols')
diff --git a/omaha_server/omaha_server/settings_test.py b/omaha_server/omaha_server/settings_test.py
index 17c4a65..d9fe6d8 100644
--- a/omaha_server/omaha_server/settings_test.py
+++ b/omaha_server/omaha_server/settings_test.py
@@ -54,4 +54,5 @@ CACHES['statistics'] = {
 
 OMAHA_UID_KEY_PREFIX = 'test:uid'
 
-CRASH_SYMBOLS_PATH = os.path.join(BASE_DIR, 'crash', 'tests', 'testdata', 'symbols')
\ No newline at end of file
+CRASH_SYMBOLS_PATH = os.path.join(BASE_DIR, 'crash', 'tests', 'testdata', 'symbols')
+CRASH_S3_MOUNT_PATH = os.path.join(BASE_DIR, 'crash', 'tests', 'testdata')
\ No newline at end of file
diff --git a/pavement.py b/pavement.py
index 51d6f7d..46dc858 100644
--- a/pavement.py
+++ b/pavement.py
@@ -18,6 +18,8 @@ License for the specific language governing permissions and limitations under
 the License.
 """
 
+import os
+
 from paver.easy import task
 from paver.easy import sh
 
@@ -67,11 +69,22 @@ def create_admin():
 
 
 @task
+def mount_s3():
+    kwargs = dict(bucket=os.environ['AWS_STORAGE_BUCKET_NAME'],
+                  mount_point='/srv/omaha_s3')
+    env = dict(AWSACCESSKEYID=os.environ['AWS_ACCESS_KEY_ID'],
+               AWSSECRETACCESSKEY=os.environ['AWS_SECRET_ACCESS_KEY'])
+    cmd = 's3fs {bucket} {mount_point} -ouse_cache=/tmp'.format(**kwargs)
+    sh(cmd, env=env)
+
+
+@task
 def docker_run():
     migrate()
     loaddata()
     create_admin()
     collectstatic()
+    mount_s3()
     sh('/usr/bin/supervisord')
 
 
diff --git a/requirements.txt b/requirements.txt
index a26a5c8..18cb7b1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+six==1.8.0
 Django==1.7.1
 django-extensions==1.4.6
 django-suit==0.2.12
@@ -20,6 +21,7 @@ msgpack-python
 django-tables2==0.15.0
 Django-Select2==4.2.2
 clom==0.7.5
+furl==0.4.0
 
 # Only dev
 django-httplog==0.2.2
\ No newline at end of file