User:MPopov (WMF)/Wikimania 2021 Hackathon

This page is for Graph-powered data visualizations of Wikidata Query Service (WDQS) usage (from 2015 to April 2021) as a replacement for soon-to-be decommissioned Discovery Dashboards – specifically the WDQS dashboard (see T227782 for more details). This project was completed by Mikhail Popov (aka bearloga) for Wikimania 2021 Hackathon.

Data
The original TSV was processed using R and uploaded as tabular datasets to Data namespace on Wikimedia Commons:
 * Data:Sandbox/MPopov_(WMF)/WDQS_homepage_traffic.tab
 * Data:Sandbox/MPopov_(WMF)/WDQS_SPARQL_endpoint.tab
 * Data:Sandbox/MPopov_(WMF)/WDQS_LDF_endpoint.tab

The code used to process and re-format the data can be found here.

Visualizations
Blue represents manual human traffic and orange is automated machine-to-machine traffic.

Homepage traffic
Visits made to query.wikidata.org.

 { "version": 2, "width": 800, "height": 400, "padding": "strict", "data": [ {     "name": "labels", "values": [ { "is_automata": true, "label": "Automated/programmatic" }, { "is_automata": false, "label": "Detected as user" } ]   },    {      "name": "visits", "url": "tabular:///Sandbox/MPopov_(WMF)/WDQS_homepage_traffic.tab", "format": { "type": "json", "property": "data", "parse": { "date": "date:'%Y-%m-%d'" }     },      "transform": [ {         "type": "filter", "test": "datum.http_success == true" },       {          "type": "lookup", "on": "labels", "onKey": "is_automata", "keys": ["is_automata"], "as": ["labels"], "default": null }     ]    }  ],  "signals": [ {"name": "lastDate", "init": {"expr": "time('2021-04-25')"}}, {     "name": "indexDate", "init": {"expr": "time('Aug 15 2015')"}, "streams": [{ "type": "mousemove", "expr": "clamp(eventX, 0, eventGroup('root').width)", "scale": {"name": "x", "invert": true} }]   }  ],  "scales": [ {     "name": "x", "type": "time", "domain": { "data": "visits", "field": "date" },     "range": "width" },   {      "name": "y", "type": "log", "domain": { "data": "visits", "field": "requests" },     "range": "height" },   {      "name": "color", "type": "ordinal", "domain": { "data": "visits", "field": "is_automata" },     "range": "category10" } ],  "axes": [ {     "type": "y", "scale": "y", "title": "Requests (log10 scale)", "grid": true, "layer": "back" },   {      "type": "x", "scale": "x", "grid": false, "layer": "back", "formatType": "time", "format": "%Y" } ],  "marks": [ {     "type": "group", "from": { "data": "visits", "transform": [{"type": "facet", "groupby": ["is_automata"]}] },     "marks": [ {         "type": "line", "properties": { "update": { "x": {"scale": "x", "field": "date"}, "y": {"scale": "y", "field": "requests"}, "stroke": {"scale": "color", "field": "is_automata"}, "strokeWidth": {"value": 1} }         }        },        {          "type": "text", "from": { "transform": [{"type": "filter", "test": "datum.date > lastDate"}] },         "properties": { "update": { "x": {"scale": "x", "field": "date", "offset": 2}, "y": {"scale": "y", "field": "requests"}, "fill": {"scale": "color", "field": "is_automata"}, "text": {"field": "labels.label"}, "baseline": {"value": "middle"} }         }        }      ]    },    {      "type":"rule", "properties": { "update": { "x": {"scale": "x", "signal": "indexDate"}, "y": {"value": 0}, "y2": {"field": {"group": "height"}}, "stroke": {"value": "black"} }     }    },    {      "type":"text", "properties": { "update": { "x": {"scale": "x", "signal": "indexDate"}, "y2": {"value": 0, "offset": -5}, "align": {"value": "center"}, "text": {"template": "\u007B{indexDate | time: '%e %B %Y'}}"}, "fill": {"value": "black"} }     }    }  ] }

SPARQL endpoint
Requests made to the SPARQL endpoint

 { "version": 2, "width": 800, "height": 400, "padding": "strict", "data": [ {     "name": "labels", "values": [ { "is_automata": true, "label": "Automated/programmatic" }, { "is_automata": false, "label": "Detected as user" } ]   },    {      "name": "visits", "url": "tabular:///Sandbox/MPopov_(WMF)/WDQS_SPARQL_endpoint.tab", "format": { "type": "json", "property": "data", "parse": { "date": "date:'%Y-%m-%d'" }     },      "transform": [ {         "type": "filter", "test": "datum.http_success == true" },       {          "type": "lookup", "on": "labels", "onKey": "is_automata", "keys": ["is_automata"], "as": ["labels"], "default": null }     ]    }  ],  "signals": [ {"name": "lastDate", "init": {"expr": "time('2021-04-25')"}}, {     "name": "indexDate", "init": {"expr": "time('Aug 15 2015')"}, "streams": [{ "type": "mousemove", "expr": "clamp(eventX, 0, eventGroup('root').width)", "scale": {"name": "x", "invert": true} }]   }  ],  "scales": [ {     "name": "x", "type": "time", "domain": { "data": "visits", "field": "date" },     "range": "width" },   {      "name": "y", "type": "log", "domain": { "data": "visits", "field": "requests" },     "range": "height" },   {      "name": "color", "type": "ordinal", "domain": { "data": "visits", "field": "is_automata" },     "range": "category10" } ],  "axes": [ {     "type": "y", "scale": "y", "title": "Requests (log10 scale)", "grid": true, "layer": "back" },   {      "type": "x", "scale": "x", "grid": false, "layer": "back", "formatType": "time", "format": "%Y" } ],  "marks": [ {     "type": "group", "from": { "data": "visits", "transform": [{"type": "facet", "groupby": ["is_automata"]}] },     "marks": [ {         "type": "line", "properties": { "update": { "x": {"scale": "x", "field": "date"}, "y": {"scale": "y", "field": "requests"}, "stroke": {"scale": "color", "field": "is_automata"}, "strokeWidth": {"value": 1} }         }        },        {          "type": "text", "from": { "transform": [{"type": "filter", "test": "datum.date > lastDate"}] },         "properties": { "update": { "x": {"scale": "x", "field": "date", "offset": 2}, "y": {"scale": "y", "field": "requests"}, "fill": {"scale": "color", "field": "is_automata"}, "text": {"field": "labels.label"}, "baseline": {"value": "middle"} }         }        }      ]    },    {      "type":"rule", "properties": { "update": { "x": {"scale": "x", "signal": "indexDate"}, "y": {"value": 0}, "y2": {"field": {"group": "height"}}, "stroke": {"value": "black"} }     }    },    {      "type":"text", "properties": { "update": { "x": {"scale": "x", "signal": "indexDate"}, "y2": {"value": 0, "offset": -5}, "align": {"value": "center"}, "text": {"template": "\u007B{indexDate | time: '%e %B %Y'}}"}, "fill": {"value": "black"} }     }    }  ] }

LDF endpoint
Requests made to the Linked Data Fragments (LDF) endpoint

 { "version": 2, "width": 800, "height": 400, "padding": "strict", "data": [ {     "name": "labels", "values": [ { "is_automata": true, "label": "Automated/programmatic" }, { "is_automata": false, "label": "Detected as user" } ]   },    {      "name": "visits", "url": "tabular:///Sandbox/MPopov_(WMF)/WDQS_LDF_endpoint.tab", "format": { "type": "json", "property": "data", "parse": { "date": "date:'%Y-%m-%d'" }     },      "transform": [ {         "type": "filter", "test": "datum.http_success == true" },       {          "type": "lookup", "on": "labels", "onKey": "is_automata", "keys": ["is_automata"], "as": ["labels"], "default": null }     ]    }  ],  "signals": [ {"name": "lastDate", "init": {"expr": "time('2021-04-25')"}}, {     "name": "indexDate", "init": {"expr": "time('Jan 1 2017')"}, "streams": [{ "type": "mousemove", "expr": "clamp(eventX, 0, eventGroup('root').width)", "scale": {"name": "x", "invert": true} }]   }  ],  "scales": [ {     "name": "x", "type": "time", "domain": { "data": "visits", "field": "date" },     "range": "width" },   {      "name": "y", "type": "linear", "domain": { "data": "visits", "field": "requests" },     "range": "height" },   {      "name": "color", "type": "ordinal", "domain": { "data": "visits", "field": "is_automata" },     "range": "category10" } ],  "axes": [ {     "type": "y", "scale": "y", "title": "Requests", "grid": true, "layer": "back" },   {      "type": "x", "scale": "x", "grid": false, "layer": "back", "formatType": "time", "format": "%Y" } ],  "marks": [ {     "type": "group", "from": { "data": "visits", "transform": [{"type": "facet", "groupby": ["is_automata"]}] },     "marks": [ {         "type": "line", "properties": { "update": { "x": {"scale": "x", "field": "date"}, "y": {"scale": "y", "field": "requests"}, "stroke": {"scale": "color", "field": "is_automata"}, "strokeWidth": {"value": 1} }         }        },        {          "type": "text", "from": { "transform": [{"type": "filter", "test": "datum.date > lastDate"}] },         "properties": { "update": { "x": {"scale": "x", "field": "date", "offset": 2}, "y": {"scale": "y", "field": "requests"}, "fill": {"scale": "color", "field": "is_automata"}, "text": {"field": "labels.label"}, "baseline": {"value": "middle"} }         }        }      ]    },    {      "type":"rule", "properties": { "update": { "x": {"scale": "x", "signal": "indexDate"}, "y": {"value": 0}, "y2": {"field": {"group": "height"}}, "stroke": {"value": "black"} }     }    },    {      "type":"text", "properties": { "update": { "x": {"scale": "x", "signal": "indexDate"}, "y2": {"value": 0, "offset": -5}, "align": {"value": "center"}, "text": {"template": "\u007B{indexDate | time: '%e %B %Y'}}"}, "fill": {"value": "black"} }     }    }  ] }