diff options
| -rw-r--r-- | omaha_server/omaha/api.py | 3 | ||||
| -rwxr-xr-x | omaha_server/omaha/management/commands/generate_fake_statistics.py | 28 | ||||
| -rw-r--r-- | omaha_server/omaha/static/statistics/js/month_charts.js | 7 | ||||
| -rw-r--r-- | omaha_server/omaha/statistics.py | 34 | ||||
| -rw-r--r-- | omaha_server/omaha/tests/fixtures.py | 24 | ||||
| -rw-r--r-- | omaha_server/omaha/tests/test_api.py | 31 | ||||
| -rw-r--r-- | omaha_server/omaha/tests/test_core.py | 4 | ||||
| -rw-r--r-- | omaha_server/omaha/tests/test_statistics.py | 143 | ||||
| -rw-r--r-- | omaha_server/omaha/tests/test_views.py | 12 | ||||
| -rw-r--r-- | omaha_server/omaha/tests/utils.py | 13 | ||||
| -rw-r--r-- | omaha_server/sparkle/statistics.py | 34 | ||||
| -rw-r--r-- | omaha_server/sparkle/tests/test_statistics.py | 148 |
12 files changed, 312 insertions, 169 deletions
diff --git a/omaha_server/omaha/api.py b/omaha_server/omaha/api.py index 9301e43..e0f36c4 100644 --- a/omaha_server/omaha/api.py +++ b/omaha_server/omaha/api.py @@ -192,12 +192,13 @@ class StatisticsMonthsDetailView(APIView): diapasons = [((start.month if year == start.year else 1, end.month if year == end.year else 12), year) for year in range(start.year, end.year+1)] - win_data = dict(new=[], updates=[]) + win_data = dict(new=[], updates=[], uninstalls=[]) mac_data = dict(new=[], updates=[]) for diapason in diapasons: step = get_users_statistics_months(app_id=app.id, platform='win', year=diapason[1], start=diapason[0][0], end=diapason[0][1]) win_data['new'] += step['new'] win_data['updates'] += step['updates'] + win_data['uninstalls'] += step['uninstalls'] step = get_users_statistics_months(app_id=app.id, platform='mac', year=diapason[1], start=diapason[0][0], end=diapason[0][1]) mac_data['new'] += step['new'] mac_data['updates'] += step['updates'] diff --git a/omaha_server/omaha/management/commands/generate_fake_statistics.py b/omaha_server/omaha/management/commands/generate_fake_statistics.py index ee479c4..16e2e9b 100755 --- a/omaha_server/omaha/management/commands/generate_fake_statistics.py +++ b/omaha_server/omaha/management/commands/generate_fake_statistics.py @@ -18,6 +18,7 @@ License for the specific language governing permissions and limitations under the License. """ +import __builtin__ import random from datetime import datetime from uuid import uuid4 @@ -30,7 +31,16 @@ from django.core.management.base import BaseCommand from omaha.models import Version, Channel from omaha.statistics import userid_counting +from omaha.tests.utils import create_app_xml from sparkle.models import SparkleVersion +from sparkle.statistics import userid_counting as mac_userid_counting + + +events = [dict(eventtype="2", eventresult="1", errorcode="0", extracode1="0"), + dict(eventtype="2", eventresult="1", errorcode="0", extracode1="0"), + dict(eventtype="2", eventresult="1", errorcode="0", extracode1="0"), + dict(eventtype="2", eventresult="1", errorcode="0", extracode1="0"), + dict(eventtype="4", eventresult="1", errorcode="0", extracode1="0")] NUMBER_UNIQUE = 1000 uuids = dict( @@ -38,6 +48,8 @@ uuids = dict( mac=["{%s}" % uuid4() for i in range(NUMBER_UNIQUE)] ) +COUNTING = dict(win=userid_counting, + mac=mac_userid_counting) def get_random_uuid(platform): return random.choice(uuids[platform]) @@ -52,22 +64,22 @@ def generate_statistics(i, versions, channels, year): platform = version.platform.name if getattr(version, 'platform', None) else 'mac' channel = random.choice(channels) if platform == 'win': - app_list = [dict( - appid=version.app.id, - version=str(version.version), - tag=channel, - )] + app = create_app_xml(appid=version.app.id, + version=str(version.version), + tag=channel.name, + events=[random.choice(events)]) + app_list = [app] else: - app_list = [dict( + app_list = dict( appid=version.app.id, version=str(version.short_version), tag=version.channel - )] + ) month = random.choice(range(1, 13)) day = random.choice(range(1, 28)) date = datetime(year, month, day) userid = get_random_uuid(platform) - userid_counting(userid, app_list, platform, now=date) + COUNTING[platform](userid, app_list, platform, now=date) def run_worker(data, versions, channels, year): diff --git a/omaha_server/omaha/static/statistics/js/month_charts.js b/omaha_server/omaha/static/statistics/js/month_charts.js index f48ffcd..d997a4f 100644 --- a/omaha_server/omaha/static/statistics/js/month_charts.js +++ b/omaha_server/omaha/static/statistics/js/month_charts.js @@ -26,14 +26,15 @@ function getData(data){ - var tmp; + if (data.hasOwnProperty('uninstalls')) + data.uninstalls.forEach(function(el, index, arr){arr[index][1] = -el[1]}); var res = Object.keys(data).map(function(d){ return { key: d, values: data[d] } }); - tmp = res[0]; res[0] = res[1]; res[1] = tmp; + res.unshift(res.pop()); return res; } @@ -56,7 +57,7 @@ .y(function(d) { return d[1] }) .stacked(true) .showControls(false) - .color(['#00f', 'green']); + .color(['#00f', 'green', 'red']); var graphSelector = ''.concat('#', platform, '-months-chart svg'); chart.xAxis.showMaxMin(false) diff --git a/omaha_server/omaha/statistics.py b/omaha_server/omaha/statistics.py index 414ec44..cd20e4c 100644 --- a/omaha_server/omaha/statistics.py +++ b/omaha_server/omaha/statistics.py @@ -53,19 +53,30 @@ def add_app_statistics(userid, platform, app, now=None): appid = app.get('appid') version = app.get('version') channel = app.get('tag') or DEFAULT_CHANNEL - + events = app.findall('event') + err_events = filter(lambda x: x.get('eventresult') not in ['1', '2', '3'], events) + if err_events: + return + install_event = filter(lambda x: x.get('eventtype') == '2', events) if is_new_install(appid, userid): - mark('new_install:%s' % appid, userid) - mark('new_install:{}:{}'.format(appid, platform), userid) - redis.setbit("known_users:%s" % appid, userid, 1) + if install_event and install_event[0].get('eventresult') in ['1', '2', '3']: + mark('new_install:%s' % appid, userid) + mark('new_install:{}:{}'.format(appid, platform), userid) + redis.setbit("known_users:%s" % appid, userid, 1) elif userid not in MonthEvents('new_install:{}:{}'.format(appid, platform), year=now.year, month=now.month): mark('request:%s' % appid, userid) mark('request:{}:{}'.format(appid, platform), userid) + uninstall_event = filter(lambda x: x.get('eventtype') == '4', events) + if uninstall_event and uninstall_event[0].get('eventresult') in ['1', '2', '3']: + mark('uninstall:%s' % appid, userid) + mark('uninstall:{}:{}'.format(appid, platform), userid) + mark('request:{}:{}'.format(appid, version), userid) mark('request:{}:{}'.format(appid, channel), userid) mark('request:{}:{}:{}'.format(appid, platform, version), userid) + def update_live_statistics(userid, apps_list, platform, now=None): id = get_id(userid) list(map(partial(add_app_live_statistics, id, platform, now=now), apps_list or [])) @@ -80,13 +91,13 @@ def add_app_live_statistics(userid, platform, app, now=None): nextversion = app.get('nextversion') install_event = filter(lambda x: x.get('eventtype') == '2', events) - if install_event and install_event[0].get('eventresult') == '1': + if install_event and install_event[0].get('eventresult') in ['1', '2', '3']: mark('online:{}:{}'.format(appid, nextversion), userid) mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid) return update_event = filter(lambda x: x.get('eventtype') == '3', events) - if update_event and update_event[0].get('eventresult') == '1': + if update_event and update_event[0].get('eventresult') in ['1', '2', '3']: unmark('online:{}:{}'.format(appid, version), userid) # necessary for unmark('online:{}:{}:{}'.format(appid, platform, version), userid) # 1 hour interval mark('online:{}:{}'.format(appid, nextversion), userid) @@ -98,6 +109,7 @@ def add_app_live_statistics(userid, platform, app, now=None): mark('online:{}:{}'.format(appid, version), userid) mark('online:{}:{}:{}'.format(appid, platform, version), userid) + def get_users_statistics_months(app_id, platform=None, year=None, start=1, end=12): now = timezone.now() if not year: @@ -106,18 +118,26 @@ def get_users_statistics_months(app_id, platform=None, year=None, start=1, end=1 if platform: install_event_name = 'new_install:{}:{}'.format(app_id, platform) update_event_name = 'request:{}:{}'.format(app_id, platform) + uninstall_event_name = 'uninstall:{}:{}'.format(app_id, platform) else: install_event_name = 'new_install:%s' % app_id update_event_name = 'request:%s' % app_id + uninstall_event_name = 'uninstall:%s' % app_id installs_by_month = [] updates_by_month = [] + uninstalls_by_month = [] for m in range(start, end + 1): installs_by_month.append(MonthEvents(install_event_name, year, m)) updates_by_month.append(MonthEvents(update_event_name, year, m)) + uninstalls_by_month.append(MonthEvents(uninstall_event_name, year, m)) installs_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(installs_by_month)] updates_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(updates_by_month)] - return dict(new=installs_data, updates=updates_data) + res = dict(new=installs_data, updates=updates_data) + if platform != 'mac': + uninstalls_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(uninstalls_by_month)] + res.update(dict(uninstalls=uninstalls_data)) + return res def get_users_statistics_weeks(app_id=None): diff --git a/omaha_server/omaha/tests/fixtures.py b/omaha_server/omaha/tests/fixtures.py index 3df6216..4be086e 100644 --- a/omaha_server/omaha/tests/fixtures.py +++ b/omaha_server/omaha/tests/fixtures.py @@ -41,9 +41,9 @@ request_update_check = b"""<?xml version="1.0" encoding="UTF-8"?> </request>""" request_event = b"""<?xml version="1.0" encoding="UTF-8"?> -<request protocol="3.0" version="1.3.23.0" ismachine="1" sessionid="{2882CF9B-D9C2-4edb-9AAF-8ED5FCF366F7}" userid="{F25EC606-5FC2-449b-91FF-FA21CADB46E4}" installsource="otherinstallcmd" testsource="ossdev" requestid="{164FC0EC-8EF7-42cb-A49D-474E20E8D352}"> +<request protocol="3.0" version="1.3.23.0" ismachine="1" sessionid="{2882CF9B-D9C2-4edb-9AAF-8ED5FCF366F7}" userid="{D0BBD725-742D-44ae-8D46-0231E881D58E}" installsource="otherinstallcmd" testsource="ossdev" requestid="{164FC0EC-8EF7-42cb-A49D-474E20E8D352}"> <os platform="win" version="6.1" sp="" arch="x64"/> - <app appid="{8A69D345-D564-463C-AFF1-A69D9E530F96}" version="" nextversion="13.0.782.112" lang="en" brand="" client="" installage="6"> + <app appid="{D0AB2EBC-931B-4013-9FEB-C9C4C2225C8C}" version="" nextversion="13.0.782.112" lang="en" brand="" client="" installage="6"> <event eventtype="9" eventresult="1" errorcode="0" extracode1="0"/> <event eventtype="5" eventresult="1" errorcode="0" extracode1="0"/> <event eventtype="2" eventresult="4" errorcode="-2147219440" extracode1="268435463"/> @@ -52,9 +52,9 @@ request_event = b"""<?xml version="1.0" encoding="UTF-8"?> """ request_event_install_success = b"""<?xml version="1.0" encoding="UTF-8"?> -<request protocol="3.0" version="1.3.23.0" ismachine="1" sessionid="{2882CF9B-D9C2-4edb-9AAF-8ED5FCF366F7}" userid="{F25EC606-5FC2-449b-91FF-FA21CADB46E4}" installsource="otherinstallcmd" testsource="ossdev" requestid="{164FC0EC-8EF7-42cb-A49D-474E20E8D352}"> +<request protocol="3.0" version="1.3.23.0" ismachine="1" sessionid="{2882CF9B-D9C2-4edb-9AAF-8ED5FCF366F7}" userid="{D0BBD725-742D-44ae-8D46-0231E881D58E}" installsource="otherinstallcmd" testsource="ossdev" requestid="{164FC0EC-8EF7-42cb-A49D-474E20E8D352}"> <os platform="win" version="6.1" sp="" arch="x64"/> - <app appid="{8A69D345-D564-463C-AFF1-A69D9E530F96}" version="" nextversion="0.0.0.1" lang="en" brand="" client="" installage="6"> + <app appid="{D0AB2EBC-931B-4013-9FEB-C9C4C2225C8C}" version="" nextversion="0.0.0.1" lang="en" brand="" client="" installage="6"> <event eventtype="9" eventresult="1" errorcode="0" extracode1="0"/> <event eventtype="5" eventresult="1" errorcode="0" extracode1="0"/> <event eventtype="2" eventresult="1" errorcode="0" extracode1="0"/> @@ -63,9 +63,9 @@ request_event_install_success = b"""<?xml version="1.0" encoding="UTF-8"?> """ request_event_update_success = b"""<?xml version="1.0" encoding="UTF-8"?> -<request protocol="3.0" version="1.3.23.0" ismachine="1" sessionid="{2882CF9B-D9C2-4edb-9AAF-8ED5FCF366F7}" userid="{F25EC606-5FC2-449b-91FF-FA21CADB46E4}" installsource="otherinstallcmd" testsource="ossdev" requestid="{164FC0EC-8EF7-42cb-A49D-474E20E8D352}"> +<request protocol="3.0" version="1.3.23.0" ismachine="1" sessionid="{2882CF9B-D9C2-4edb-9AAF-8ED5FCF366F7}" userid="{D0BBD725-742D-44ae-8D46-0231E881D58E}" installsource="otherinstallcmd" testsource="ossdev" requestid="{164FC0EC-8EF7-42cb-A49D-474E20E8D352}"> <os platform="win" version="6.1" sp="" arch="x64"/> - <app appid="{8A69D345-D564-463C-AFF1-A69D9E530F96}" version="0.0.0.1" nextversion="0.0.0.2" lang="en" brand="" client="" installage="6"> + <app appid="{D0AB2EBC-931B-4013-9FEB-C9C4C2225C8C}" version="0.0.0.1" nextversion="0.0.0.2" lang="en" brand="" client="" installage="6"> <event eventtype="9" eventresult="1" errorcode="0" extracode1="0"/> <event eventtype="5" eventresult="1" errorcode="0" extracode1="0"/> <event eventtype="3" eventresult="1" errorcode="0" extracode1="0"/> @@ -74,9 +74,9 @@ request_event_update_success = b"""<?xml version="1.0" encoding="UTF-8"?> """ request_event_uninstall_success = b"""<?xml version="1.0" encoding="UTF-8"?> -<request protocol="3.0" version="1.3.23.0" ismachine="1" sessionid="{2882CF9B-D9C2-4edb-9AAF-8ED5FCF366F7}" userid="{F25EC606-5FC2-449b-91FF-FA21CADB46E4}" installsource="otherinstallcmd" testsource="ossdev" requestid="{164FC0EC-8EF7-42cb-A49D-474E20E8D352}"> +<request protocol="3.0" version="1.3.23.0" ismachine="1" sessionid="{2882CF9B-D9C2-4edb-9AAF-8ED5FCF366F7}" userid="{D0BBD725-742D-44ae-8D46-0231E881D58E}" installsource="otherinstallcmd" testsource="ossdev" requestid="{164FC0EC-8EF7-42cb-A49D-474E20E8D352}"> <os platform="win" version="6.1" sp="" arch="x64"/> - <app appid="{8A69D345-D564-463C-AFF1-A69D9E530F96}" version="0.0.0.2" nextversion="" lang="en" brand="" client="" installage="6"> + <app appid="{D0AB2EBC-931B-4013-9FEB-C9C4C2225C8C}" version="0.0.0.2" nextversion="" lang="en" brand="" client="" installage="6"> <event eventtype="9" eventresult="1" errorcode="0" extracode1="0"/> <event eventtype="5" eventresult="1" errorcode="0" extracode1="0"/> <event eventtype="4" eventresult="1" errorcode="0" extracode1="0"/> @@ -137,7 +137,7 @@ response_update_check_positive = b"""<?xml version="1.0" encoding="UTF-8"?> response_event = b"""<?xml version="1.0" encoding="UTF-8"?> <response protocol="3.0" server="prod"> <daystart elapsed_seconds="56754"/> - <app appid="{8A69D345-D564-463C-AFF1-A69D9E530F96}" status="ok"> + <app appid="{D0AB2EBC-931B-4013-9FEB-C9C4C2225C8C}" status="ok"> <event status="ok"/> <event status="ok"/> <event status="ok"/> @@ -147,7 +147,7 @@ response_event = b"""<?xml version="1.0" encoding="UTF-8"?> response_data_doc = b"""<?xml version="1.0" encoding="UTF-8"?> <response protocol="3.0" server="prod"> <daystart elapsed_seconds="56754"/> - <app appid="{8A69D345-D564-463C-AFF1-A69D9E530F96}" status="ok"> + <app appid="{D0AB2EBC-931B-4013-9FEB-C9C4C2225C8C}" status="ok"> <data index="verboselogging" name="install" status="ok"> app-specific values here </data> @@ -174,3 +174,7 @@ response_data = b"""<?xml version="1.0" encoding="UTF-8"?> <ping status="ok"/> </app> </response>""" + +event_install_success = dict(eventtype="2", eventresult="1", errorcode="0", extracode1="0") +event_install_error = dict(eventtype="2", eventresult="0", errorcode="0", extracode1="0") +event_uninstall_success = dict(eventtype="4", eventresult="1", errorcode="0", extracode1="0")
\ No newline at end of file diff --git a/omaha_server/omaha/tests/test_api.py b/omaha_server/omaha/tests/test_api.py index ac79eb7..7b8abc0 100644 --- a/omaha_server/omaha/tests/test_api.py +++ b/omaha_server/omaha/tests/test_api.py @@ -29,6 +29,7 @@ from django.contrib.auth import get_user_model from django.core.files.uploadedfile import SimpleUploadedFile from django.conf import settings +from lxml.builder import E from rest_framework import status from rest_framework.test import APITestCase, APIClient from bitmapist import mark_event @@ -50,9 +51,10 @@ from omaha.serializers import ( ) from omaha.factories import ApplicationFactory, DataFactory, PlatformFactory, ChannelFactory, VersionFactory, ActionFactory from omaha.models import Application, Data, Channel, Platform, Version, Action -from omaha.tests.utils import temporary_media_root +from omaha.tests import fixtures +from omaha.tests.utils import temporary_media_root, create_app_xml from sparkle.models import SparkleVersion - +from sparkle.statistics import userid_counting as mac_userid_counting User = get_user_model() @@ -307,7 +309,6 @@ class LiveStatistics(APITestCase): self.assertDictEqual(StatisticsMonthsSerializer(self.data).data, response.data) - class StatisticsMonthsMixin(object): url = None url_args = () @@ -322,14 +323,16 @@ class StatisticsMonthsMixin(object): date = datetime(year=prev_year, month=i, day=10) for id in range(1, i + 1): user_id = UUID(int=id) - userid_counting(user_id, self.app_list, self.platform.name, now=date) + userid_counting(user_id, self.install_app_list, self.platform.name, now=date) user_id = UUID(int=1000 + id) - userid_counting(user_id, [self.mac_app], 'mac', now=date) + mac_userid_counting(user_id, self.mac_app, 'mac', now=date) + userid_counting(UUID(int=i), self.uninstall_app_list, self.platform.name, now=date) user_id = UUID(int=13) - userid_counting(user_id, self.app_list, self.platform.name, now=datetime(year=now.year, month=1, day=1)) + userid_counting(user_id, self.install_app_list, self.platform.name, now=datetime(year=now.year, month=1, day=1)) + userid_counting(user_id, self.uninstall_app_list, self.platform.name, now=datetime(year=now.year, month=1, day=1)) user_id = UUID(int=1013) - userid_counting(user_id, [self.mac_app], 'mac', now=datetime(year=now.year, month=1, day=1)) + mac_userid_counting(user_id, self.mac_app, 'mac', now=datetime(year=now.year, month=1, day=1)) @freeze_time("2016-01-27") @temporary_media_root() @@ -362,7 +365,11 @@ class StatisticsMonthsMixin(object): dsa_signature='MCwCFCdoW13VBGJWIfIklKxQVyetgxE7AhQTVuY9uQT0KOV1UEk21epBsGZMPg==', file=SimpleUploadedFile('./chrome.dmg', b'_' * 1024), file_size=1024) - self.app_list = [dict(appid=self.app.id, version=str(self.version1.version))] + app_kwargs = dict(appid=self.app.id, version=str(self.version1.version)) + install_app = create_app_xml(events=[fixtures.event_install_success], **app_kwargs) + uninstall_app = create_app_xml(events=[fixtures.event_uninstall_success], **app_kwargs) + self.install_app_list = [install_app] + self.uninstall_app_list = [uninstall_app] self.mac_app = dict(appid=self.app.id, version=str(self.mac_version.short_version)) self._generate_fake_statistics() @@ -371,8 +378,12 @@ class StatisticsMonthsMixin(object): updates.append((datetime(now.year, 1, 1).strftime("%Y-%m"), 0)) installs = [(datetime(now.year-1, x, 1).strftime("%Y-%m"), 1) for x in range(2, 13)] installs.append((datetime(now.year, 1, 1).strftime("%Y-%m"), 1)) - platform_statistics = dict(new=installs, updates=updates) - self.users_statistics = dict(win=platform_statistics, mac=platform_statistics) + uninstalls = [(datetime(now.year-1, x, 1).strftime("%Y-%m"), 1) for x in range(2, 13)] + uninstalls.append((datetime(now.year, 1, 1).strftime("%Y-%m"), 1)) + + win_platform_statistics = dict(new=installs, updates=updates, uninstalls=uninstalls) + mac_platform_statistics = dict(new=installs, updates=updates) + self.users_statistics = dict(win=win_platform_statistics, mac=mac_platform_statistics) self.data = dict(data=dict(self.users_statistics)) @is_private() diff --git a/omaha_server/omaha/tests/test_core.py b/omaha_server/omaha/tests/test_core.py index 4b6216d..f442b99 100644 --- a/omaha_server/omaha/tests/test_core.py +++ b/omaha_server/omaha/tests/test_core.py @@ -267,7 +267,7 @@ class TestRequestScheme(TestCase, XmlTestMixin): response = Response( date=datetime(year=2014, month=1, day=1, hour=15, minute=45, second=54), apps_list=[App( - app_id='{8A69D345-D564-463C-AFF1-A69D9E530F96}', + app_id='{D0AB2EBC-931B-4013-9FEB-C9C4C2225C8C}', status='ok', events=[Event(), Event(), Event()] )] @@ -280,7 +280,7 @@ class TestRequestScheme(TestCase, XmlTestMixin): response = Response( date=datetime(year=2014, month=1, day=1, hour=15, minute=45, second=54), apps_list=[App( - app_id='{8A69D345-D564-463C-AFF1-A69D9E530F96}', + app_id='{D0AB2EBC-931B-4013-9FEB-C9C4C2225C8C}', status='ok', data_list=[ Data('install', index='verboselogging', text='app-specific values here'), diff --git a/omaha_server/omaha/tests/test_statistics.py b/omaha_server/omaha/tests/test_statistics.py index 82abd06..bff2b98 100644 --- a/omaha_server/omaha/tests/test_statistics.py +++ b/omaha_server/omaha/tests/test_statistics.py @@ -50,7 +50,7 @@ from omaha.statistics import ( get_users_versions, ) -from omaha.tests.utils import temporary_media_root +from omaha.tests.utils import temporary_media_root, create_app_xml from omaha.utils import redis, get_id from omaha.settings import DEFAULT_CHANNEL from omaha.models import ( @@ -68,6 +68,7 @@ from omaha.models import ( from sparkle.models import SparkleVersion from sparkle.statistics import userid_counting as mac_userid_counting + class StatisticsTest(TestCase): def setUp(self): redis.flushdb() @@ -116,40 +117,89 @@ class StatisticsTest(TestCase): self.assertEqual(len(request_events), 2) + @freeze_time('2016-1-1') def test_add_app_statistics(self): now = datetime.utcnow() + next_month = now.replace(month=now.month + 1) userid = 1 channel = DEFAULT_CHANNEL platform = 'win' - app = dict(appid='{F97917B1-20AB-48C1-9802-CEF305B10804}', version='30.0.123.1234') - appid = app.get('appid') - version = app.get('version') - - events_appid = DayEvents('new_install:%s' % app.get('appid'), now.year, now.month, now.day) - events_appid_version = DayEvents('request:{}:{}'.format(appid, version), now.year, now.month, now.day) - events_appid_platform = DayEvents('new_install:{}:{}'.format(appid, platform), now.year, now.month, now.day) - events_appid_channel = DayEvents('request:{}:{}'.format(appid, channel), now.year, now.month, now.day) - events_appid_platform_version = DayEvents('request:{}:{}:{}'.format(appid, platform, version), now.year, now.month, now.day) - - self.assertEqual(len(events_appid), 0) - self.assertEqual(len(events_appid_version), 0) - self.assertEqual(len(events_appid_platform), 0) - self.assertEqual(len(events_appid_channel), 0) - self.assertEqual(len(events_appid_platform_version), 0) - - add_app_statistics(userid, platform, app) - - self.assertEqual(len(events_appid), 1) - self.assertEqual(len(events_appid_version), 1) - self.assertEqual(len(events_appid_platform), 1) - self.assertEqual(len(events_appid_channel), 1) - self.assertEqual(len(events_appid_platform_version), 1) - - self.assertIn(userid, events_appid) - self.assertIn(userid, events_appid_version) - self.assertIn(userid, events_appid_platform) - self.assertIn(userid, events_appid_channel) - self.assertIn(userid, events_appid_platform_version) + app_kwargs = dict(appid='{F97917B1-20AB-48C1-9802-CEF305B10804}', version='30.0.123.1234') + success_app = create_app_xml(events=fixtures.event_install_success, **app_kwargs) + error_app = create_app_xml(events=fixtures.event_install_error, **app_kwargs) + appid = app_kwargs.get('appid') + version = app_kwargs.get('version') + + events_request_appid = lambda date=now: DayEvents.from_date('request:%s' % appid, date) + events_new_appid = lambda date=now: DayEvents.from_date('new_install:%s' % appid, date) + events_request_appid_version = lambda date=now: DayEvents.from_date('request:{}:{}'.format(appid, version), date) + events_request_appid_platform = lambda date=now: DayEvents.from_date('request:{}:{}'.format(appid, platform), date) + events_new_appid_platform = lambda date=now: DayEvents.from_date('new_install:{}:{}'.format(appid, platform), date) + events_request_appid_channel = lambda date=now: DayEvents.from_date('request:{}:{}'.format(appid, channel), date) + events_request_appid_platform_version = lambda date=now: DayEvents.from_date('request:{}:{}:{}'.format(appid, platform, version), date) + + + self.assertEqual(len(events_new_appid()), 0) + self.assertEqual(len(events_request_appid()), 0) + self.assertEqual(len(events_request_appid_version()), 0) + self.assertEqual(len(events_request_appid_platform()), 0) + self.assertEqual(len(events_new_appid_platform()), 0) + self.assertEqual(len(events_request_appid_channel()), 0) + self.assertEqual(len(events_request_appid_platform_version()), 0) + + add_app_statistics(userid, platform, error_app) + + self.assertEqual(len(events_new_appid()), 0) + self.assertEqual(len(events_request_appid()), 0) + self.assertEqual(len(events_request_appid_version()), 0) + self.assertEqual(len(events_request_appid_platform()), 0) + self.assertEqual(len(events_new_appid_platform()), 0) + self.assertEqual(len(events_request_appid_channel()), 0) + self.assertEqual(len(events_request_appid_platform_version()), 0) + + add_app_statistics(userid, platform, success_app) + self.assertEqual(len(events_new_appid()), 1) + self.assertEqual(len(events_request_appid()), 0) + self.assertEqual(len(events_request_appid_version()), 1) + self.assertEqual(len(events_new_appid_platform()), 1) + self.assertEqual(len(events_request_appid_platform()), 0) + self.assertEqual(len(events_request_appid_channel()), 1) + self.assertEqual(len(events_request_appid_platform_version()), 1) + + self.assertIn(userid, events_new_appid()) + self.assertIn(userid, events_request_appid_version()) + self.assertIn(userid, events_new_appid_platform()) + self.assertIn(userid, events_request_appid_channel()) + self.assertIn(userid, events_request_appid_platform_version()) + + add_app_statistics(userid, platform, success_app) + self.assertEqual(len(events_new_appid()), 1) + self.assertEqual(len(events_request_appid()), 0) + self.assertEqual(len(events_request_appid_version()), 1) + self.assertEqual(len(events_new_appid_platform()), 1) + self.assertEqual(len(events_request_appid_platform()), 0) + self.assertEqual(len(events_request_appid_channel()), 1) + self.assertEqual(len(events_request_appid_platform_version()), 1) + + with freeze_time(next_month): + add_app_statistics(userid, platform, error_app) + + self.assertEqual(len(events_request_appid(next_month)), 0) + self.assertEqual(len(events_request_appid_platform(next_month)), 0) + + with freeze_time(next_month): + add_app_statistics(userid, platform, success_app) + + self.assertEqual(len(events_request_appid(next_month)), 1) + self.assertEqual(len(events_request_appid_platform(next_month)), 1) + self.assertEqual(len(events_new_appid(next_month)), 0) + self.assertEqual(len(events_request_appid_version(next_month)), 1) + self.assertEqual(len(events_new_appid_platform(next_month)), 0) + self.assertEqual(len(events_request_appid_channel()), 1) + self.assertEqual(len(events_request_appid_platform_version()), 1) + + self.assertIn(userid, events_request_appid(next_month)) + self.assertIn(userid, events_request_appid_platform(next_month)) def test_add_app_live_statistics(self): request = parse_request(fixtures.request_update_check) @@ -231,7 +281,7 @@ class StatisticsTest(TestCase): self.assertEqual(req.version, Request._meta.get_field_by_name('version')[0].to_python('1.3.23.0')) self.assertEqual(req.ismachine, 1) self.assertEqual(req.sessionid, '{2882CF9B-D9C2-4edb-9AAF-8ED5FCF366F7}') - self.assertEqual(req.userid, '{F25EC606-5FC2-449b-91FF-FA21CADB46E4}') + self.assertEqual(req.userid, '{D0BBD725-742D-44ae-8D46-0231E881D58E}') self.assertEqual(req.originurl, None) self.assertEqual(req.testsource, 'ossdev') self.assertEqual(req.updaterchannel, None) @@ -252,7 +302,7 @@ class StatisticsTest(TestCase): self.assertEqual(app.lang, 'en') self.assertEqual(app.tag, None) self.assertEqual(app.installage, 6) - self.assertEqual(app.appid, '{8A69D345-D564-463C-AFF1-A69D9E530F96}') + self.assertEqual(app.appid, '{D0AB2EBC-931B-4013-9FEB-C9C4C2225C8C}') def test_parse_events(self): request = parse_request(fixtures.request_event) @@ -307,7 +357,7 @@ class StatisticsTest(TestCase): self.assertEqual(req.version, Request._meta.get_field_by_name('version')[0].to_python('1.3.23.0')) self.assertEqual(req.ismachine, 1) self.assertEqual(req.sessionid, '{2882CF9B-D9C2-4edb-9AAF-8ED5FCF366F7}') - self.assertEqual(req.userid, '{F25EC606-5FC2-449b-91FF-FA21CADB46E4}') + self.assertEqual(req.userid, '{D0BBD725-742D-44ae-8D46-0231E881D58E}') self.assertEqual(req.originurl, None) self.assertEqual(req.testsource, 'ossdev') self.assertEqual(req.updaterchannel, None) @@ -319,7 +369,7 @@ class StatisticsTest(TestCase): self.assertEqual(app_req.lang, 'en') self.assertEqual(app_req.tag, None) self.assertEqual(app_req.installage, 6) - self.assertEqual(app_req.appid, '{8A69D345-D564-463C-AFF1-A69D9E530F96}') + self.assertEqual(app_req.appid, '{D0AB2EBC-931B-4013-9FEB-C9C4C2225C8C}') self.assertEqual(app_req.request, req) event = events[0] @@ -420,6 +470,8 @@ class StatisticsTest(TestCase): class GetStatisticsTest(TestCase): + maxDiff = None + def _generate_fake_statistics(self): now = datetime.now() year = now.year @@ -429,9 +481,11 @@ class GetStatisticsTest(TestCase): date = datetime(year=year, month=i, day=10) for id in range(1, i + 1): user_id = UUID(int=id) - userid_counting(user_id, self.app_list, self.platform.name, now=date) + userid_counting(user_id, self.install_app_list, self.platform.name, now=date) user_id = UUID(int=n_users + id) - userid_counting(user_id, [self.mac_app], 'mac', now=date) + mac_userid_counting(user_id, self.mac_app, 'mac', now=date) + userid_counting(UUID(int=i), self.uninstall_app_list, self.platform.name, now=date) + @temporary_media_root() def setUp(self): @@ -459,21 +513,28 @@ class GetStatisticsTest(TestCase): dsa_signature='MCwCFCdoW13VBGJWIfIklKxQVyetgxE7AhQTVuY9uQT0KOV1UEk21epBsGZMPg==', file=SimpleUploadedFile('./chrome.dmg', b'_' * 23963192), file_size=23963192) - self.app_list = [dict(appid=self.app.id, version=str(self.version1.version))] + + app_kwargs = dict(appid=self.app.id, version=str(self.version1.version)) + install_app = create_app_xml(events=[fixtures.event_install_success], **app_kwargs) + uninstall_app = create_app_xml(events=[fixtures.event_uninstall_success], **app_kwargs) + self.install_app_list = [install_app] + self.uninstall_app_list = [uninstall_app] self.mac_app = dict(appid=self.app.id, version=str(self.mac_version.short_version)) self._generate_fake_statistics() now = datetime.now() win_updates = [(datetime(now.year, x, 1).strftime("%Y-%m"), x - 1) for x in range(1, 13)] win_installs = [(datetime(now.year, x, 1).strftime("%Y-%m"), 1) for x in range(1, 13)] + uninstalls = [(datetime(now.year, x, 1).strftime("%Y-%m"), 1) for x in range(1, 13)] mac_updates = [(datetime(now.year, x, 1).strftime("%Y-%m"), x - 1) for x in range(1, 13)] mac_installs = [(datetime(now.year, x, 1).strftime("%Y-%m"), 1) for x in range(1, 13)] total_installs = map(lambda x, y: (x[0], x[1] + y[1]), win_installs, mac_installs) total_updates = map(lambda x, y: (x[0], x[1] + y[1]), win_updates, mac_updates) - self.users_statistics = dict(new=total_installs, updates=total_updates) - self.win_users_statistics = dict(new=win_installs, updates=win_updates) + self.users_statistics = dict(new=total_installs, updates=total_updates, uninstalls=uninstalls) + self.win_users_statistics = dict(new=win_installs, updates=win_updates, uninstalls=uninstalls) self.mac_users_statistics = dict(new=mac_installs, updates=mac_updates) - + + def tearDown(self): redis.flushdb() @@ -486,7 +547,7 @@ class GetStatisticsTest(TestCase): def test_get_chanels_statistics(self): now = datetime.now() with freeze_time(datetime(year=now.year, month=now.month, day=10)): - self.assertListEqual(get_channel_statistics(self.app.id), [('stable', now.month * 2)]) + self.assertListEqual(get_channel_statistics(self.app.id), [('stable', now.month*2)]) def test_get_users_versions(self): now = datetime.now() diff --git a/omaha_server/omaha/tests/test_views.py b/omaha_server/omaha/tests/test_views.py index 504c5c3..48eb417 100644 --- a/omaha_server/omaha/tests/test_views.py +++ b/omaha_server/omaha/tests/test_views.py @@ -151,21 +151,25 @@ class UpdateViewTest(TestCase, XmlTestMixin): with freeze_time(install_date): # 56508 sec self.client.post(reverse('update'), fixtures.request_update_check, content_type='text/xml') + self.client.post(reverse('update'), + fixtures.request_event_install_success, content_type='text/xml') self.assertEqual(len(request_events), 1) - self.assertEqual(len(app1_install_events), 1) + self.assertEqual(len(app1_install_events), 0) self.assertEqual(len(app2_install_events), 1) + self.assertEqual(len(app1_update_events), 0) + self.assertEqual(len(app2_update_events), 0) self.assertTrue(user_id in request_events) - self.assertTrue(user_id in app1_install_events) + self.assertFalse(user_id in app1_install_events) self.assertTrue(user_id in app2_install_events) with freeze_time(update_date): self.client.post(reverse('update'), fixtures.request_update_check, content_type='text/xml') - self.assertEqual(len(app1_update_events), 1) + self.assertEqual(len(app1_update_events), 0) self.assertEqual(len(app2_update_events), 1) - self.assertTrue(user_id in app1_update_events) + self.assertFalse(user_id in app1_update_events) self.assertTrue(user_id in app2_update_events) @freeze_time('2014-01-01 15:45:54') # 56754 sec diff --git a/omaha_server/omaha/tests/utils.py b/omaha_server/omaha/tests/utils.py index f499879..8058097 100644 --- a/omaha_server/omaha/tests/utils.py +++ b/omaha_server/omaha/tests/utils.py @@ -23,6 +23,7 @@ import tempfile from django.conf import settings from django.test import override_settings +from lxml.builder import E class temporary_media_root(override_settings): """Temporarily override settings.MEDIA_ROOT with a temporary directory. @@ -65,3 +66,15 @@ class temporary_media_root(override_settings): setting.""" shutil.rmtree(settings.MEDIA_ROOT) super(temporary_media_root, self).disable() + + +def create_app_xml(**kwargs): + events = kwargs.pop('events', []) + app = dict(**kwargs) + app = E.app(app) + if type(events) is not list: + events = [events] + for event in events: + e = E.event(event) + app.append(e) + return app diff --git a/omaha_server/sparkle/statistics.py b/omaha_server/sparkle/statistics.py index 08d2769..debabc2 100644 --- a/omaha_server/sparkle/statistics.py +++ b/omaha_server/sparkle/statistics.py @@ -20,9 +20,12 @@ the License. from functools import partial -from bitmapist import mark_event +from bitmapist import mark_event, MonthEvents +from django.utils import timezone + +from omaha.statistics import get_id, is_new_install, redis +from omaha.settings import DEFAULT_CHANNEL -from omaha.statistics import get_id, userid_counting def collect_statistics(request, appid, channel): deviceID = request.GET.get('deviceID') @@ -33,7 +36,7 @@ def collect_statistics(request, appid, channel): version=version, tag=channel) - userid_counting(deviceID, [app], 'mac') + userid_counting(deviceID, app, 'mac') update_live_statistics(deviceID, appid, version) @@ -44,3 +47,28 @@ def update_live_statistics(userid, appid, version, now=None): mark('online:{}:{}:{}'.format(appid, 'mac', version), userid) +def userid_counting(userid, app, platform, now=None): + id = get_id(userid) + mark_event('request', id, now=now) + add_app_statistics(id, platform, app, now=now) + + +def add_app_statistics(userid, platform, app, now=None): + mark = partial(mark_event, now=now) + if not now: + now = timezone.now() + appid = app.get('appid') + version = app.get('version') + channel = app.get('tag') or DEFAULT_CHANNEL + + if is_new_install(appid, userid): + mark('new_install:%s' % appid, userid) + mark('new_install:{}:{}'.format(appid, platform), userid) + redis.setbit("known_users:%s" % appid, userid, 1) + elif userid not in MonthEvents('new_install:{}:{}'.format(appid, platform), year=now.year, month=now.month): + mark('request:%s' % appid, userid) + mark('request:{}:{}'.format(appid, platform), userid) + + mark('request:{}:{}'.format(appid, version), userid) + mark('request:{}:{}'.format(appid, channel), userid) + mark('request:{}:{}:{}'.format(appid, platform, version), userid)
\ No newline at end of file diff --git a/omaha_server/sparkle/tests/test_statistics.py b/omaha_server/sparkle/tests/test_statistics.py index 8d4ce1b..efd017d 100644 --- a/omaha_server/sparkle/tests/test_statistics.py +++ b/omaha_server/sparkle/tests/test_statistics.py @@ -19,16 +19,20 @@ the License. """ from django.test import TestCase, RequestFactory +from datetime import datetime + import mock from django_redis import get_redis_connection from bitmapist import DayEvents, HourEvents +from freezegun import freeze_time from omaha.utils import get_id from sparkle.statistics import update_live_statistics, collect_statistics -from omaha.statistics import add_app_statistics +from sparkle.statistics import add_app_statistics redis = get_redis_connection('statistics') + class StatisticsTest(TestCase): request_factory = RequestFactory() @@ -38,91 +42,75 @@ class StatisticsTest(TestCase): def tearDown(self): redis.flushdb() + @freeze_time('2016-1-1') def test_add_app_statistics(self): - userid1 = '{F07B3878-CD6F-4B96-B52F-95C4D23077E0}' - user1_id = get_id(userid1) - - userid2 = '{EC4C5647-F798-4BCA-83DA-926CD448A1D5}' - user2_id = get_id(userid2) - + now = datetime.utcnow() + next_month = now.replace(month=now.month + 1) + userid = 1 + platform = 'mac' appid = '{F97917B1-19AB-48C1-9802-CEF305B10804}' version = '0.0.0.1' channel = 'test' test_app = dict(appid=appid, version=version, tag=channel) - install_app_events = DayEvents('new_install:%s' % appid) - request_app_events = DayEvents('request:%s' % appid) - request_version_events = DayEvents('request:{}:{}'.format(appid, version)) - install_platform_events = DayEvents('new_install:{}:{}'.format(appid, 'mac')) - request_platform_events = DayEvents('request:{}:{}'.format(appid, 'mac')) - request_channel_events = DayEvents('request:{}:{}'.format(appid, channel)) - request_platform_version_events = DayEvents('request:{}:{}:{}'.format(appid, 'mac', version)) - - self.assertFalse(user1_id in install_app_events) - self.assertEqual(len(install_app_events), 0) - self.assertFalse(user1_id in install_platform_events) - self.assertEqual(len(install_platform_events), 0) - self.assertFalse(user1_id in request_app_events) - self.assertEqual(len(request_app_events), 0) - self.assertFalse(user1_id in request_platform_events) - self.assertEqual(len(request_platform_events), 0) - self.assertFalse(user1_id in request_version_events) - self.assertEqual(len(request_version_events), 0) - self.assertFalse(user1_id in request_channel_events) - self.assertEqual(len(request_channel_events), 0) - self.assertFalse(user1_id in request_platform_version_events) - self.assertEqual(len(request_platform_version_events), 0) - - add_app_statistics(user1_id, 'mac', test_app) - self.assertTrue(user1_id in install_app_events) - self.assertEqual(len(install_app_events), 1) - self.assertFalse(user1_id in request_app_events) - self.assertEqual(len(request_app_events), 0) - self.assertFalse(user1_id in request_platform_events) - self.assertEqual(len(request_platform_events), 0) - self.assertTrue(user1_id in request_version_events) - self.assertEqual(len(request_version_events), 1) - self.assertTrue(user1_id in install_platform_events) - self.assertEqual(len(install_platform_events), 1) - self.assertTrue(user1_id in request_channel_events) - self.assertEqual(len(request_channel_events), 1) - self.assertTrue(user1_id in request_platform_version_events) - self.assertEqual(len(request_platform_version_events), 1) - - - add_app_statistics(user2_id, 'mac', test_app) - - self.assertTrue(user2_id in install_app_events) - self.assertEqual(len(install_app_events), 2) - self.assertFalse(user2_id in request_app_events) - self.assertEqual(len(request_app_events), 0) - self.assertFalse(user2_id in request_platform_events) - self.assertEqual(len(request_platform_events), 0) - self.assertTrue(user2_id in request_version_events) - self.assertEqual(len(request_version_events), 2) - self.assertTrue(user2_id in install_platform_events) - self.assertEqual(len(install_platform_events), 2) - self.assertTrue(user2_id in request_channel_events) - self.assertEqual(len(request_channel_events), 2) - self.assertTrue(user2_id in request_platform_version_events) - self.assertEqual(len(request_platform_version_events), 2) - - add_app_statistics(user1_id, 'mac', test_app) - - self.assertTrue(user1_id in install_app_events) - self.assertEqual(len(install_app_events), 2) - self.assertFalse(user1_id in request_app_events) - self.assertEqual(len(request_app_events), 0) - self.assertFalse(user1_id in request_platform_events) - self.assertEqual(len(request_platform_events), 0) - self.assertTrue(user1_id in request_version_events) - self.assertEqual(len(request_version_events), 2) - self.assertTrue(user1_id in install_platform_events) - self.assertEqual(len(install_platform_events), 2) - self.assertTrue(user1_id in request_channel_events) - self.assertEqual(len(request_channel_events), 2) - self.assertTrue(user1_id in request_platform_version_events) - self.assertEqual(len(request_platform_version_events), 2) + events_request_appid = lambda date=now: DayEvents.from_date('request:%s' % appid, date) + events_new_appid = lambda date=now: DayEvents.from_date('new_install:%s' % appid, date) + events_request_appid_version = lambda date=now: DayEvents.from_date('request:{}:{}'.format(appid, version), + date) + events_request_appid_platform = lambda date=now: DayEvents.from_date('request:{}:{}'.format(appid, platform), + date) + events_new_appid_platform = lambda date=now: DayEvents.from_date('new_install:{}:{}'.format(appid, platform), + date) + events_request_appid_channel = lambda date=now: DayEvents.from_date('request:{}:{}'.format(appid, channel), + date) + events_request_appid_platform_version = lambda date=now: DayEvents.from_date( + 'request:{}:{}:{}'.format(appid, platform, version), date) + + self.assertEqual(len(events_new_appid()), 0) + self.assertEqual(len(events_request_appid()), 0) + self.assertEqual(len(events_request_appid_version()), 0) + self.assertEqual(len(events_request_appid_platform()), 0) + self.assertEqual(len(events_new_appid_platform()), 0) + self.assertEqual(len(events_request_appid_channel()), 0) + self.assertEqual(len(events_request_appid_platform_version()), 0) + + add_app_statistics(userid, platform, test_app) + self.assertEqual(len(events_new_appid()), 1) + self.assertEqual(len(events_request_appid()), 0) + self.assertEqual(len(events_request_appid_version()), 1) + self.assertEqual(len(events_new_appid_platform()), 1) + self.assertEqual(len(events_request_appid_platform()), 0) + self.assertEqual(len(events_request_appid_channel()), 1) + self.assertEqual(len(events_request_appid_platform_version()), 1) + + self.assertIn(userid, events_new_appid()) + self.assertIn(userid, events_request_appid_version()) + self.assertIn(userid, events_new_appid_platform()) + self.assertIn(userid, events_request_appid_channel()) + self.assertIn(userid, events_request_appid_platform_version()) + + add_app_statistics(userid, platform, test_app) + self.assertEqual(len(events_new_appid()), 1) + self.assertEqual(len(events_request_appid()), 0) + self.assertEqual(len(events_request_appid_version()), 1) + self.assertEqual(len(events_new_appid_platform()), 1) + self.assertEqual(len(events_request_appid_platform()), 0) + self.assertEqual(len(events_request_appid_channel()), 1) + self.assertEqual(len(events_request_appid_platform_version()), 1) + + with freeze_time(next_month): + add_app_statistics(userid, platform, test_app) + + self.assertEqual(len(events_request_appid(next_month)), 1) + self.assertEqual(len(events_request_appid_platform(next_month)), 1) + self.assertEqual(len(events_new_appid(next_month)), 0) + self.assertEqual(len(events_request_appid_version(next_month)), 1) + self.assertEqual(len(events_new_appid_platform(next_month)), 0) + self.assertEqual(len(events_request_appid_channel()), 1) + self.assertEqual(len(events_request_appid_platform_version()), 1) + + self.assertIn(userid, events_request_appid(next_month)) + self.assertIn(userid, events_request_appid_platform(next_month)) def test_update_live_statistics(self): userid1 = '{F07B3878-CD6F-4B96-B52F-95C4D23077E0}' |