beta_release_os_gfx.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unable to parse whitelist (/home/hadoop/anaconda2/lib/python2.7/site-packages/moztelemetry/bucket-whitelist.json). Assuming all histograms are acceptable.\n",
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hadoop/anaconda2/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    }
   ],
   "source": [
    "import json, time, sys\n",
    "import tarfile\n",
    "import os\n",
    "import urllib2\n",
    "import re\n",
    "from cStringIO import StringIO\n",
    "from pprint import pprint\n",
    "from moztelemetry import get_pings, get_pings_properties, get_one_ping_per_client\n",
    "\n",
    "%pylab inline\n",
    "\n",
    "data_url= 'https://analysis-output.telemetry.mozilla.org/beta_release_os_gfx/data'\n",
    "\n",
    "# This dir is auto created when the script is scheduled\n",
    "!mkdir -p output\n",
    "\n",
    "## !!! EDIT THESE BEFORE RUNNING !!!\n",
    "##\n",
    "## These represent the number of uniform random samples of raw\n",
    "## sessions we'll take, from each channel, as a fraction.\n",
    "##\n",
    "## When testing on small clusters, it is important to keep the\n",
    "## fraction low. Using a cluster of size 4, without about 100,000\n",
    "## pings, the job below took me about 10 minutes. The fraction\n",
    "## was f=0.001. For iterative development, it is best to use even\n",
    "## smaller fractions.\n",
    "##\n",
    "## As of Nov 2015, 1% (fraction=0.01) of sessions results in\n",
    "## about 1,000,000 samples for Beta, and about 1/5th that for\n",
    "## Release. After FHR is removed, the Release population will\n",
    "## become much, much larger.\n",
    "channels = {'beta': {'fraction': 0.3}, 'release': {'fraction': 0.3}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "windows_map = { (3, 5): 'NT',\n",
    "                (4, 0): '95',\n",
    "                (4, 1): '98',\n",
    "                (4, 10): '98',\n",
    "                (4, 9): 'Me',\n",
    "                (4, 90): 'Me',\n",
    "                (5, 0): '2000',\n",
    "                (5, 1): 'XP',\n",
    "                (5, 2): 'XP 64',\n",
    "                (6, 0): 'Vista',\n",
    "                (6, 1): '7',\n",
    "                (6, 2): '8',\n",
    "                (6, 3): '8.1',\n",
    "                (10, 0): '10' }\n",
    "\n",
    "mac_map = { 15: '10.11.x El Capitan',\n",
    "            14: '10.10.x Yosemite',\n",
    "            13: '10.9.x',\n",
    "            12: '10.8.x',\n",
    "            11: '10.7.x',\n",
    "            10: '10.6.x',\n",
    "            9: '10.5.x',\n",
    "            8: '10.4.x' }\n",
    "\n",
    "def get_prettyname(name, number):\n",
    "    if name.startswith('Windows'):\n",
    "        return get_prettyname_win(number)\n",
    "    elif name.startswith('Darwin'):\n",
    "        return get_prettyname_mac(number)\n",
    "    elif name.startswith('Linux'):\n",
    "        return get_prettyname_linux(number)\n",
    "    else:\n",
    "        return 'Unknown'\n",
    "\n",
    "    \n",
    "def get_prettyname_win(number):\n",
    "    versions = number.split('.')\n",
    "    if versions and versions[0].isdigit():\n",
    "        major = int(versions[0])\n",
    "        if len(versions) >= 2 and versions[1].isdigit():\n",
    "            minor = int(versions[1])\n",
    "        else:\n",
    "            minor = 0\n",
    "        return 'Windows %s' % windows_map.get((major, minor), 'unknown')\n",
    "    return 'Windows unknown'\n",
    "\n",
    "\n",
    "def get_prettyname_mac(number):\n",
    "    versions = number.split('.')\n",
    "    if versions and versions[0].isdigit():\n",
    "        major = int(versions[0])\n",
    "        return 'Mac OS X %s' % mac_map.get(major, 'unknown')\n",
    "    return 'Mac OS X unknown'\n",
    "\n",
    "\n",
    "def get_prettyname_linux(number):\n",
    "    return 'Linux'\n",
    "\n",
    "\n",
    "def get_base_os_name(name):\n",
    "    if name.startswith('Windows'):\n",
    "        return 'Windows'\n",
    "    elif name.startswith('Darwin'):\n",
    "        return 'Mac OS X'\n",
    "    elif name.startswith('Linux'):\n",
    "        return 'Linux'\n",
    "    else:\n",
    "        return 'Unknown'    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class pciids(object):\n",
    "\n",
    "    __pciids = {}\n",
    "\n",
    "    @staticmethod\n",
    "    def __get_ids():\n",
    "        if not pciids.__pciids:\n",
    "            pciids.__get_data_from_pciids()\n",
    "            pciids.__get_data_from_pcidatabase()\n",
    "            pciids.__get_data_byhand()\n",
    "        return pciids.__pciids\n",
    "\n",
    "    \n",
    "    @staticmethod\n",
    "    def __get_data_from_pciids():\n",
    "        url = 'https://raw.githubusercontent.com/pciutils/pciids/master/pci.ids'\n",
    "        response = urllib2.urlopen(url)\n",
    "        data = response.read()\n",
    "        response.close()\n",
    "\n",
    "        data = data.split('\\n')\n",
    "        \n",
    "        # Syntax:\n",
    "        # vendor  vendor_name\n",
    "        #\tdevice  device_name\t\t\t\t<-- single tab\n",
    "        #\t\tsubvendor subdevice  subsystem_name\t<-- two tabs\n",
    "        for line in data:\n",
    "            if line and not line.startswith('#'):\n",
    "                if not line.startswith('\\t'): # vendor  vendor_name\n",
    "                    line = line.strip()\n",
    "                    pos = line.index('  ')\n",
    "                    vendorID = line[:pos]\n",
    "                    vendorName = line[(pos + 2):]\n",
    "                    devices = {}\n",
    "                    pciids.__pciids[vendorID] = { 'name': vendorName, 'devices': devices }\n",
    "                elif not line.startswith('\\t\\t'): # device  device_name\n",
    "                    line = line.strip()\n",
    "                    pos = line.index('  ')\n",
    "                    deviceID = line[:pos]\n",
    "                    deviceName = line[(pos + 2):]\n",
    "                    devices[deviceID] = deviceName\n",
    "            elif line and line == '# List of known device classes, subclasses and programming interfaces':\n",
    "                break\n",
    "\n",
    "            \n",
    "    @staticmethod\n",
    "    def __get_data_from_pcidatabase():\n",
    "        url = 'http://pcidatabase.com/reports.php?type=csv'\n",
    "        response = urllib2.urlopen(url)\n",
    "        data = response.read()\n",
    "        response.close()\n",
    "\n",
    "        #\"0x0033\",\"0x002F\",\"Paradyne Corp.\",\"MAIAM\",\"Spitfire VGA Accelerator\"\n",
    "        pat = re.compile('\\\"0x([0-9a-fA-F]{4})\\\",\\\"0x([0-9a-fA-F]{4})\\\",\\\"([^\\\"]*)\\\",\\\"([^\\\"]*)\\\",\\\"([^\\\"]*)\\\"')\n",
    "\n",
    "        data = data.split('\\n')\n",
    "        for line in data:\n",
    "            if line:\n",
    "                for m in pat.finditer(line):\n",
    "                    vendorID = m.group(1).lower()\n",
    "                    deviceID = m.group(2).lower()\n",
    "                    vendorName = m.group(3)\n",
    "                    desc1 = m.group(4)\n",
    "                    desc2 = m.group(5)\n",
    "                    desc = desc2 if desc2 else desc1\n",
    "                    desc = desc if desc else '%s Unknown' % vendorName \n",
    "                    if vendorID in pciids.__pciids:\n",
    "                        devices = pciids.__pciids[vendorID]['devices']\n",
    "                        if deviceID not in devices:\n",
    "                            devices[deviceID] = desc\n",
    "                    else:\n",
    "                        pciids.__pciids[vendorID] = { 'name': vendorName, 'devices': { deviceID: desc } }\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def __get_data_byhand():\n",
    "        pciids.__pciids['8086']['devices']['22b1'] = 'Intel(R) HD Graphics'\n",
    "        pciids.__pciids['8086']['devices']['0d22'] = 'Intel(R) Iris(TM) Pro Graphics 5200'\n",
    "        pciids.__pciids['8086']['devices']['22b0'] = 'Intel(R) HD Graphics'\n",
    "        pciids.__pciids['8086']['devices']['4102'] = 'Intel(R) Graphics Media Accelerator 600'\n",
    "        pciids.__pciids['1022']['devices']['68f9'] = 'AMD Radeon Graphics Processor'\n",
    "        pciids.__pciids['1002']['devices']['6920'] = 'AMD RADEON R9 M395X'\n",
    "        pciids.__pciids['1002']['devices']['714f'] = 'RV505'\n",
    "        pciids.__pciids['10de']['devices']['0a6b'] = 'NVIDIA 9400 GT 512 MB BIOS'\n",
    "        pciids.__pciids['1414']['devices']['02c1'] = 'Microsoft RemoteFX Graphics Device - WDDM'\n",
    "        pciids.__pciids['1414']['devices']['008c'] = 'Microsoft Basic Render Driver'\n",
    "        pciids.__pciids['1414']['devices']['fefe'] = '0xfefe'\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def get_vendor_name(vendorID):\n",
    "        vendor = pciids.__get_ids().get(vendorID, None)\n",
    "        if vendor:\n",
    "            return vendor['name']\n",
    "        return 'Unknown'\n",
    "\n",
    "    \n",
    "    @staticmethod\n",
    "    def get_pci_desc(vendorID, deviceID):\n",
    "        vendor = pciids.__get_ids().get(vendorID, None)\n",
    "        if vendor:\n",
    "            desc = vendor['devices'].get(deviceID, None)\n",
    "            if desc:\n",
    "                return desc\n",
    "            else:\n",
    "                return '%s - Unknown' % vendor['name']\n",
    "        else:\n",
    "            return 'Unknown'\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def mk_id(pciid):\n",
    "        if pciid.startswith('0x'):\n",
    "            pciid = pciid[2:]\n",
    "            l = len(pciid)\n",
    "            if l < 4:\n",
    "                pciid = '0' * (4 - l) + pciid\n",
    "            elif l == 4:\n",
    "                pciid = pciid.replace(' ', '0')\n",
    "            else:\n",
    "                return ''\n",
    "            return pciid\n",
    "        return ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Disable logging, since this eats a ton of diskspace.\n",
    "def quiet_logs(sc):\n",
    "  logger = sc._jvm.org.apache.log4j\n",
    "  logger.LogManager.getLogger(\"org\").setLevel(logger.Level.ERROR)\n",
    "  logger.LogManager.getLogger(\"akka\").setLevel(logger.Level.ERROR)\n",
    "quiet_logs(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Constants.\n",
    "GfxAdaptersKey = 'environment/system/gfx/adapters'\n",
    "SystemOsKey = 'environment/system/os'\n",
    "SubmissionDate = 'meta/submissionDate'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fmt_date(d):\n",
    "    return d.strftime(\"%Y%m%d\")\n",
    "\n",
    "\n",
    "today = datetime.datetime.now()\n",
    "start_date = fmt_date(today - datetime.timedelta(2))\n",
    "end_date = start_date\n",
    "#start_date = '20160411'\n",
    "#end_date = '20160422'\n",
    "\n",
    "def get_pings_for_channel(channel, fraction):\n",
    "    date_range = (start_date, end_date)\n",
    "    \n",
    "    args = {\n",
    "        'app': 'Firefox',\n",
    "        'schema': 'v4',\n",
    "        'submission_date': date_range,\n",
    "        'channel': channel,\n",
    "        'fraction': fraction,\n",
    "    }\n",
    "    \n",
    "    pings = get_pings(sc, **args)\n",
    "    pings = get_pings_properties(pings, [\n",
    "        'clientId',\n",
    "        GfxAdaptersKey,\n",
    "        SystemOsKey,\n",
    "        SubmissionDate\n",
    "    ])\n",
    "    pings = get_one_ping_per_client(pings)\n",
    "    \n",
    "    return pings.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pings = {}\n",
    "for channel, v in channels.iteritems():\n",
    "    pings[channel] = get_pings_for_channel(channel, v['fraction'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 7544 beta pings.\n",
      "Found 107 release pings.\n"
     ]
    }
   ],
   "source": [
    "print('Found {0} beta pings.'.format(pings['beta'].count()))\n",
    "print('Found {0} release pings.'.format(pings['release'].count()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def add_1dict(x, y):\n",
    "    for k, v in y.iteritems():\n",
    "        x[k] = x[k] + v if k in x else v\n",
    "    return x\n",
    "\n",
    "def add_2dict(x, y):\n",
    "    for k, v in y.iteritems():\n",
    "        x[k] = add_1dict(x[k], v) if k in x else v\n",
    "    return x\n",
    "\n",
    "def get_os(p):\n",
    "    os = p[SystemOsKey]\n",
    "    return (os['name'], { os['version']: 1 })\n",
    "\n",
    "def get_gfx(p):\n",
    "    gfx = p[GfxAdaptersKey]\n",
    "    unk = 'Unknown'\n",
    "    if gfx:\n",
    "        vendorID = gfx[0].get('vendorID', unk)\n",
    "        deviceID = gfx[0].get('deviceID', unk)\n",
    "        driverVersion = gfx[0].get('driverVersion', unk)   \n",
    "        return (vendorID, { deviceID: { driverVersion: 1} })\n",
    "    return (unk, unk)\n",
    "\n",
    "def get_stats(pings, action, add):\n",
    "    return pings.map(action).reduceByKey(add).collect()\n",
    "\n",
    "def total(d):\n",
    "    s = 0\n",
    "    for v in d.itervalues():\n",
    "        s += v\n",
    "    return s\n",
    "\n",
    "def reduce_version(v):\n",
    "    _v = v.split('.')\n",
    "    if len(_v) >= 2 and _v[0].isdigit() and _v[1].isdigit():\n",
    "        return '%s.%s' % (_v[0], _v[1])\n",
    "    else:\n",
    "        return v\n",
    "\n",
    "\n",
    "def improve_os(oses):\n",
    "    _oses = {}\n",
    "    n = 0\n",
    "    for os in oses:\n",
    "        os_name = os[0]\n",
    "        versions = os[1]\n",
    "        t = total(versions)\n",
    "\n",
    "        base_name = get_base_os_name(os_name)\n",
    "        if base_name in _oses:\n",
    "            os = _oses[base_name]\n",
    "            os['total'] += t\n",
    "            _versions = os['versions']\n",
    "        else:\n",
    "            _versions = {} \n",
    "            _oses[base_name] = { 'total': t, 'versions': _versions }\n",
    "\n",
    "        for version, value in versions.iteritems():\n",
    "            name = get_prettyname(os_name, version)\n",
    "            _versions[name] = _versions[name] + value if name in versions else value\n",
    "        \n",
    "        n += t\n",
    "        \n",
    "    return { 'total': n, 'oses': _oses}\n",
    "\n",
    "def improve_gfx(gfxs):\n",
    "    _gfxs = {}\n",
    "    r = 0\n",
    "    for gfx in gfxs:\n",
    "        vendorID = gfx[0]\n",
    "        vendorName = pciids.get_vendor_name(pciids.mk_id(vendorID))\n",
    "        if vendorName in _gfxs:\n",
    "            _devices = _gfxs[vendorName]['devices']\n",
    "        else:\n",
    "            _devices = {}\n",
    "            _gfxs[vendorName] = { 'total': 0, 'devices': _devices }\n",
    "        \n",
    "        devices = gfx[1]\n",
    "        s = 0\n",
    "        for deviceID, versions in devices.iteritems():\n",
    "            t = total(versions)\n",
    "            s += t\n",
    "            deviceName = pciids.get_pci_desc(pciids.mk_id(vendorID), pciids.mk_id(deviceID))\n",
    "            if deviceName in _devices:\n",
    "                device = _devices[deviceName]\n",
    "                _versions = device['versions']\n",
    "                device['total'] += t\n",
    "            else:\n",
    "                _versions = {}\n",
    "                _devices[deviceName] = { 'total': t, 'versions': _versions }\n",
    "            \n",
    "            for version, value in versions.iteritems():\n",
    "                #version = reduce_version(version)\n",
    "                _versions[version] = _versions[version] + value if version in _versions else value\n",
    "                \n",
    "        _gfxs[vendorName]['total'] += s\n",
    "        r += s\n",
    "    \n",
    "    return { 'total': r, 'gfxs': _gfxs}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "os_data = { }\n",
    "for channel, p in pings.iteritems():\n",
    "    data = get_stats(p, get_os, add_1dict)\n",
    "    data = improve_os(data)\n",
    "    os_data[channel] = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gfx_data = { }\n",
    "for channel, p in pings.iteritems():\n",
    "    data = get_stats(p, get_gfx, add_2dict)\n",
    "    data = improve_gfx(data)\n",
    "    gfx_data[channel] = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = { 'start_date': start_date, 'end_date': end_date, 'platforms': os_data, 'graphics': gfx_data }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#pprint(data['graphics'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def aggregate(agg, data):\n",
    "    agg['end_date'] = data['end_date']\n",
    "    \n",
    "    # we aggregate the os\n",
    "    for channel, data_os in data['platforms'].iteritems():\n",
    "        agg_os = agg['platforms'][channel]\n",
    "        agg_oses = agg_os['oses']\n",
    "        data_oses = data_os['oses']\n",
    "        agg_os['total'] += data_os['total']\n",
    "        for osName, versions in data_oses.iteritems():\n",
    "            if osName in agg_oses:\n",
    "                agg_versions = agg_oses[osName]\n",
    "                agg_versions['total'] += versions['total']\n",
    "                add_1dict(agg_versions['versions'], versions['versions'])\n",
    "            else:\n",
    "                agg_oses[osName] = versions\n",
    "                \n",
    "    # we aggregate the gfxs\n",
    "    for channel, data_gfx in data['graphics'].iteritems():\n",
    "        agg_gfx = agg['graphics'][channel]\n",
    "        agg_gfxs = agg_gfx['gfxs']\n",
    "        data_gfxs = data_gfx['gfxs']\n",
    "        agg_gfx['total'] += data_gfx['total']\n",
    "        for vendorName, devices in data_gfxs.iteritems():\n",
    "            if vendorName in agg_gfxs:\n",
    "                agg_devices = agg_gfxs[vendorName]\n",
    "                agg_devices['total'] += devices['total']\n",
    "                agg_devices = agg_devices['devices']\n",
    "                devices = devices['devices']\n",
    "                for deviceName, versions in devices.iteritems():\n",
    "                    if deviceName in agg_devices:\n",
    "                        agg_versions = agg_devices[deviceName]\n",
    "                        agg_versions['total'] += versions['total']\n",
    "                        add_1dict(agg_versions['versions'], versions['versions'])\n",
    "                    else:\n",
    "                        agg_versions[deviceName] = versions\n",
    "            else:\n",
    "                agg_gfxs[vendorName] = devices\n",
    "\n",
    "#{'beta': {'gfxs': {'Advanced Micro Devices, Inc. [AMD/ATI]': {'devices': {'BeaverCreek [Radeon HD 6520G]': {'total': 1,\n",
    "#                                                                                                            'versions': {u'8.901': 1}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    response = urllib2.urlopen('%s/agg_data.tar.gz' % data_url)\n",
    "    tgz = response.read()\n",
    "    response.close()\n",
    "except:\n",
    "    tgz = None\n",
    "\n",
    "if tgz:\n",
    "    tar = tarfile.open(mode='r:gz', fileobj=StringIO(tgz))\n",
    "    tar.extractall()\n",
    "    tar.close()\n",
    "    with open('agg_data.json') as In:\n",
    "        agg_data = json.load(In)\n",
    "        aggregate(agg_data, data)\n",
    "    with open('agg_data.json', 'w') as Out:\n",
    "        json.dump(agg_data, Out)\n",
    "else:\n",
    "    with open('agg_data.json', 'w') as Out:\n",
    "        json.dump(data, Out)\n",
    "    \n",
    "tar = tarfile.open('./output/agg_data.tar.gz', 'w:gz', compresslevel=9)\n",
    "tar.add('agg_data.json', arcname='agg_data.json')\n",
    "tar.close()\n",
    "os.remove('agg_data.json')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}