Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 6f0bded

Browse files
authoredAug 3, 2017
Merge pull request #1924 from plotly/aggregateby
Aggregate transforms
2 parents df5face + 572f282 commit 6f0bded

File tree

8 files changed

+759
-24
lines changed

8 files changed

+759
-24
lines changed
 

‎lib/aggregate.js‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
/**
2+
* Copyright 2012-2017, Plotly, Inc.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the MIT license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
'use strict';
10+
11+
module.exports = require('../src/transforms/aggregate');

‎lib/index.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ Plotly.register([
5656
// https://github.com/plotly/plotly.js/pull/978#pullrequestreview-2403353
5757
//
5858
Plotly.register([
59+
require('./aggregate'),
5960
require('./filter'),
6061
require('./groupby'),
6162
require('./sort')

‎src/plots/cartesian/axes.js‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ axes.cleanPosition = function(pos, gd, axRef) {
124124
return cleanPos(pos);
125125
};
126126

127-
axes.getDataToCoordFunc = function(gd, trace, target, targetArray) {
127+
var getDataConversions = axes.getDataConversions = function(gd, trace, target, targetArray) {
128128
var ax;
129129

130130
// If target points to an axis, use the type we already have for that
@@ -155,15 +155,23 @@ axes.getDataToCoordFunc = function(gd, trace, target, targetArray) {
155155

156156
// if 'target' has corresponding axis
157157
// -> use setConvert method
158-
if(ax) return ax.d2c;
158+
if(ax) return {d2c: ax.d2c, c2d: ax.c2d};
159159

160160
// special case for 'ids'
161161
// -> cast to String
162-
if(d2cTarget === 'ids') return function(v) { return String(v); };
162+
if(d2cTarget === 'ids') return {d2c: toString, c2d: toString};
163163

164164
// otherwise (e.g. numeric-array of 'marker.color' or 'marker.size')
165165
// -> cast to Number
166-
return function(v) { return +v; };
166+
167+
return {d2c: toNum, c2d: toNum};
168+
};
169+
170+
function toNum(v) { return +v; }
171+
function toString(v) { return String(v); }
172+
173+
axes.getDataToCoordFunc = function(gd, trace, target, targetArray) {
174+
return getDataConversions(gd, trace, target, targetArray).d2c;
167175
};
168176

169177
// empty out types for all axes containing these traces

‎src/transforms/aggregate.js‎

Lines changed: 410 additions & 0 deletions
Large diffs are not rendered by default.

‎src/transforms/filter.js‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,13 @@ exports.attributes = {
3737
description: [
3838
'Sets the filter target by which the filter is applied.',
3939

40-
'If a string, *target* is assumed to be a reference to a data array',
40+
'If a string, `target` is assumed to be a reference to a data array',
4141
'in the parent trace object.',
4242
'To filter about nested variables, use *.* to access them.',
4343
'For example, set `target` to *marker.color* to filter',
4444
'about the marker color array.',
4545

46-
'If an array, *target* is then the data array by which the filter is applied.'
46+
'If an array, `target` is then the data array by which the filter is applied.'
4747
].join(' ')
4848
},
4949
operation: {
@@ -83,23 +83,23 @@ exports.attributes = {
8383
valType: 'any',
8484
dflt: 0,
8585
description: [
86-
'Sets the value or values by which to filter by.',
86+
'Sets the value or values by which to filter.',
8787

8888
'Values are expected to be in the same type as the data linked',
89-
'to *target*.',
89+
'to `target`.',
9090

9191
'When `operation` is set to one of',
9292
'the comparison values (' + COMPARISON_OPS + ')',
93-
'*value* is expected to be a number or a string.',
93+
'`value` is expected to be a number or a string.',
9494

9595
'When `operation` is set to one of the interval values',
9696
'(' + INTERVAL_OPS + ')',
97-
'*value* is expected to be 2-item array where the first item',
97+
'`value` is expected to be 2-item array where the first item',
9898
'is the lower bound and the second item is the upper bound.',
9999

100100
'When `operation`, is set to one of the set values',
101101
'(' + SET_OPS + ')',
102-
'*value* is expected to be an array with as many items as',
102+
'`value` is expected to be an array with as many items as',
103103
'the desired set elements.'
104104
].join(' ')
105105
},

‎src/transforms/groupby.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,12 @@ exports.attributes = {
6363
*
6464
* @param {object} transformIn
6565
* object linked to trace.transforms[i] with 'type' set to exports.name
66-
* @param {object} fullData
67-
* the plot's full data
66+
* @param {object} traceOut
67+
* the _fullData trace this transform applies to
6868
* @param {object} layout
6969
* the plot's (not-so-full) layout
70+
* @param {object} traceIn
71+
* the input data trace this transform applies to
7072
*
7173
* @return {object} transformOut
7274
* copy of transformIn that contains attribute defaults
Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
var Plotly = require('@lib/index');
2+
3+
var createGraphDiv = require('../assets/create_graph_div');
4+
var destroyGraphDiv = require('../assets/destroy_graph_div');
5+
var customMatchers = require('../assets/custom_matchers');
6+
7+
describe('aggregate', function() {
8+
var gd;
9+
10+
beforeAll(function() { jasmine.addMatchers(customMatchers);});
11+
12+
beforeEach(function() { gd = createGraphDiv(); });
13+
14+
afterEach(destroyGraphDiv);
15+
16+
it('handles all funcs for numeric data', function() {
17+
// throw in some non-numbers, they should get discarded except first/last
18+
Plotly.newPlot(gd, [{
19+
x: [1, 2, 3, 4, 'fail'],
20+
y: [1.1, 2.2, 3.3, 'nope', 5.5],
21+
marker: {
22+
size: ['2001-01-01', 0.2, 0.1, 0.4, 0.5],
23+
color: [2, 4, '', 10, 8],
24+
opacity: [0.6, 'boo', 0.2, 0.8, 1.0],
25+
line: {
26+
color: [2.2, 3.3, 4.4, 5.5, 'the end']
27+
}
28+
},
29+
transforms: [{
30+
type: 'aggregate',
31+
groups: ['a', 'b', 'a', 'a', 'a'],
32+
aggregations: [
33+
// missing array - the entry is ignored
34+
{target: '', func: 'avg'},
35+
// disabled explicitly
36+
{target: 'x', func: 'avg', enabled: false},
37+
{target: 'x', func: 'sum'},
38+
// non-numerics will not count toward numerator or denominator for avg
39+
{target: 'y', func: 'avg'},
40+
{target: 'marker.size', func: 'min'},
41+
{target: 'marker.color', func: 'max'},
42+
// marker.opacity doesn't have an entry, but it will default to first
43+
// as if it were {target: 'marker.opacity', func: 'first'},
44+
{target: 'marker.line.color', func: 'last'},
45+
// not present in data, but that's OK for count
46+
{target: 'marker.line.width', func: 'count'},
47+
// duplicate entry - discarded
48+
{target: 'x', func: 'min'}
49+
]
50+
}]
51+
}], {
52+
// log axis doesn't change how sum (or avg but not tested) works
53+
xaxis: {type: 'log'}
54+
});
55+
56+
var traceOut = gd._fullData[0];
57+
58+
expect(traceOut.x).toEqual([8, 2]);
59+
expect(traceOut.y).toBeCloseToArray([3.3, 2.2], 5);
60+
expect(traceOut.marker.size).toEqual([0.1, 0.2]);
61+
expect(traceOut.marker.color).toEqual([10, 4]);
62+
expect(traceOut.marker.opacity).toEqual([0.6, 'boo']);
63+
expect(traceOut.marker.line.color).toEqual(['the end', 3.3]);
64+
expect(traceOut.marker.line.width).toEqual([4, 1]);
65+
});
66+
67+
it('handles all funcs except sum for date data', function() {
68+
// weird cases handled in another test
69+
Plotly.newPlot(gd, [{
70+
x: ['2001-01-01', '', '2001-01-03', '2001-01-05', '2001-01-07'],
71+
y: ['1995-01-15', '2005-03-15', '1990-12-23', '2001-01-01', 'not a date'],
72+
text: ['2001-01-01 12:34', '2001-01-01 12:35', '2001-01-01 12:36', '2001-01-01 12:37', ''],
73+
hovertext: ['a', '2001-01-02', '2001-01-03', '2001-01-04', '2001-01-05'],
74+
customdata: ['2001-01', 'b', '2001-03', '2001-04', '2001-05'],
75+
transforms: [{
76+
type: 'aggregate',
77+
// groups can be any type, but until we implement binning they
78+
// will always compare as strings = so 1 === '1' === 1.0 !== '1.0'
79+
groups: [1, 2, '1', 1.0, 1],
80+
aggregations: [
81+
{target: 'x', func: 'avg'},
82+
{target: 'y', func: 'min'},
83+
{target: 'text', func: 'max'},
84+
// hovertext doesn't have a func, default to first
85+
{target: 'hovertext'},
86+
{target: 'customdata', func: 'last'},
87+
// not present in data, but that's OK for count
88+
{target: 'marker.line.width', func: 'count'},
89+
// duplicate entry - discarded
90+
{target: 'x', func: 'min'}
91+
]
92+
}]
93+
}]);
94+
95+
var traceOut = gd._fullData[0];
96+
97+
expect(traceOut.x).toEqual(['2001-01-04', undefined]);
98+
expect(traceOut.y).toEqual(['1990-12-23', '2005-03-15']);
99+
expect(traceOut.text).toEqual(['2001-01-01 12:37', '2001-01-01 12:35']);
100+
expect(traceOut.hovertext).toEqual(['a', '2001-01-02']);
101+
expect(traceOut.customdata).toEqual(['2001-05', 'b']);
102+
expect(traceOut.marker.line.width).toEqual([4, 1]);
103+
});
104+
105+
it('handles all funcs except sum and avg for category data', function() {
106+
// weird cases handled in another test
107+
Plotly.newPlot(gd, [{
108+
x: ['a', 'b', 'c', 'aa', 'd'],
109+
y: ['q', 'w', 'e', 'r', 't'],
110+
text: ['b', 'b', 'a', 'b', 'a'],
111+
hovertext: ['c', 'b', 'a', 'b', 'a'],
112+
transforms: [{
113+
type: 'aggregate',
114+
groups: [1, 2, 1, 1, 1],
115+
aggregations: [
116+
{target: 'x', func: 'min'},
117+
{target: 'y', func: 'max'},
118+
{target: 'text', func: 'last'},
119+
// hovertext doesn't have an entry, but it will default to first
120+
// not present in data, but that's OK for count
121+
{target: 'marker.line.width', func: 'count'},
122+
// duplicate entry - discarded
123+
{target: 'x', func: 'max'}
124+
]
125+
}]
126+
}], {
127+
xaxis: {categoryarray: ['aaa', 'aa', 'a', 'b', 'c']}
128+
});
129+
130+
var traceOut = gd._fullData[0];
131+
132+
// explicit order (only possible for axis data)
133+
expect(traceOut.x).toEqual(['aa', 'b']);
134+
// implied order from data
135+
expect(traceOut.y).toEqual(['t', 'w']);
136+
expect(traceOut.text).toEqual(['a', 'b']);
137+
expect(traceOut.hovertext).toEqual(['c', 'b']);
138+
expect(traceOut.marker.line.width).toEqual([4, 1]);
139+
});
140+
141+
it('allows date and category sums, and category avg, with weird output', function() {
142+
// this test is more of an FYI than anything else - it doesn't break but
143+
// these results are usually meaningless.
144+
145+
Plotly.newPlot(gd, [{
146+
x: ['2001-01-01', '2001-01-02', '2001-01-03', '2001-01-04'],
147+
y: ['a', 'b', 'b', 'c'],
148+
text: ['a', 'b', 'a', 'c'],
149+
transforms: [{
150+
type: 'aggregate',
151+
groups: [1, 1, 2, 2],
152+
aggregations: [
153+
{target: 'x', func: 'sum'},
154+
{target: 'y', func: 'sum'},
155+
{target: 'text', func: 'avg'}
156+
]
157+
}]
158+
}]);
159+
160+
var traceOut = gd._fullData[0];
161+
162+
// date sums: 1970-01-01 is "zero", there are shifts due to # of leap years
163+
// without that shift these would be 2032-01-02 and 2032-01-06
164+
expect(traceOut.x).toEqual(['2032-01-03', '2032-01-07']);
165+
// category sums: can go off the end of the category array -> gives undefined
166+
expect(traceOut.y).toEqual(['b', undefined]);
167+
// category average: can result in fractional categories -> rounds (0.5 rounds to 1)
168+
expect(traceOut.text).toEqual(['b', 'b']);
169+
});
170+
171+
it('can aggregate on an existing data array', function() {
172+
Plotly.newPlot(gd, [{
173+
x: [1, 2, 3, 4, 5],
174+
y: [2, 4, 6, 8, 10],
175+
marker: {size: [10, 10, 20, 20, 10]},
176+
transforms: [{
177+
type: 'aggregate',
178+
groups: 'marker.size',
179+
aggregations: [
180+
{target: 'x', func: 'sum'},
181+
{target: 'y', func: 'avg'}
182+
]
183+
}]
184+
}]);
185+
186+
var traceOut = gd._fullData[0];
187+
188+
expect(traceOut.x).toEqual([8, 7]);
189+
expect(traceOut.y).toBeCloseToArray([16 / 3, 7], 5);
190+
expect(traceOut.marker.size).toEqual([10, 20]);
191+
});
192+
193+
it('handles median, mode, rms, & stddev for numeric data', function() {
194+
// again, nothing is going to barf with non-numeric data, but sometimes it
195+
// won't make much sense.
196+
197+
Plotly.newPlot(gd, [{
198+
x: [1, 1, 2, 2, 1],
199+
y: [1, 2, 3, 4, 5],
200+
marker: {
201+
size: [1, 2, 3, 4, 5],
202+
line: {width: [1, 1, 2, 2, 1]},
203+
color: [1, 1, 2, 2, 1]
204+
},
205+
transforms: [{
206+
type: 'aggregate',
207+
groups: [1, 2, 1, 1, 1],
208+
aggregations: [
209+
{target: 'x', func: 'mode'},
210+
{target: 'y', func: 'median'},
211+
{target: 'marker.size', func: 'rms'},
212+
{target: 'marker.line.width', func: 'stddev', funcmode: 'population'},
213+
{target: 'marker.color', func: 'stddev'}
214+
]
215+
}]
216+
}]);
217+
218+
var traceOut = gd._fullData[0];
219+
220+
// 1 and 2 both have count of 2 in the first group,
221+
// but 2 gets to that count first
222+
expect(traceOut.x).toEqual([2, 1]);
223+
expect(traceOut.y).toBeCloseToArray([3.5, 2], 5);
224+
expect(traceOut.marker.size).toBeCloseToArray([Math.sqrt(51 / 4), 2], 5);
225+
expect(traceOut.marker.line.width).toBeCloseToArray([0.5, 0], 5);
226+
expect(traceOut.marker.color).toBeCloseToArray([Math.sqrt(1 / 3), 0], 5);
227+
});
228+
});

‎test/jasmine/tests/transform_multi_test.js‎

Lines changed: 86 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,10 @@ describe('user-defined transforms:', function() {
232232
describe('multiple transforms:', function() {
233233
'use strict';
234234

235+
var gd;
236+
237+
beforeEach(function() { gd = createGraphDiv(); });
238+
235239
var mockData0 = [{
236240
mode: 'markers',
237241
x: [1, -1, -2, 0, 1, 2, 3],
@@ -278,8 +282,6 @@ describe('multiple transforms:', function() {
278282
it('Plotly.plot should plot the transform traces', function(done) {
279283
var data = Lib.extendDeep([], mockData0);
280284

281-
var gd = createGraphDiv();
282-
283285
Plotly.plot(gd, data).then(function() {
284286
expect(gd.data.length).toEqual(1);
285287
expect(gd.data[0].x).toEqual([1, -1, -2, 0, 1, 2, 3]);
@@ -302,8 +304,6 @@ describe('multiple transforms:', function() {
302304

303305
data[0].transforms.slice().reverse();
304306

305-
var gd = createGraphDiv();
306-
307307
Plotly.plot(gd, data).then(function() {
308308
expect(gd.data.length).toEqual(1);
309309
expect(gd.data[0].x).toEqual([1, -1, -2, 0, 1, 2, 3]);
@@ -325,7 +325,6 @@ describe('multiple transforms:', function() {
325325
var data = Lib.extendDeep([], mockData0);
326326
data[0].marker = { size: 20 };
327327

328-
var gd = createGraphDiv();
329328
var dims = [2, 2];
330329

331330
Plotly.plot(gd, data).then(function() {
@@ -377,8 +376,6 @@ describe('multiple transforms:', function() {
377376
it('Plotly.extendTraces should work', function(done) {
378377
var data = Lib.extendDeep([], mockData0);
379378

380-
var gd = createGraphDiv();
381-
382379
Plotly.plot(gd, data).then(function() {
383380
expect(gd.data[0].x.length).toEqual(7);
384381
expect(gd._fullData[0].x.length).toEqual(2);
@@ -405,8 +402,6 @@ describe('multiple transforms:', function() {
405402
it('Plotly.deleteTraces should work', function(done) {
406403
var data = Lib.extendDeep([], mockData1);
407404

408-
var gd = createGraphDiv();
409-
410405
Plotly.plot(gd, data).then(function() {
411406
assertDims([2, 2, 2, 2]);
412407

@@ -425,8 +420,6 @@ describe('multiple transforms:', function() {
425420
it('toggling trace visibility should work', function(done) {
426421
var data = Lib.extendDeep([], mockData1);
427422

428-
var gd = createGraphDiv();
429-
430423
Plotly.plot(gd, data).then(function() {
431424
assertDims([2, 2, 2, 2]);
432425

@@ -446,6 +439,88 @@ describe('multiple transforms:', function() {
446439
});
447440
});
448441

442+
it('executes filter and aggregate in the order given', function() {
443+
// filter and aggregate do not commute!
444+
445+
var trace1 = {
446+
x: [0, -5, 7, 4, 5],
447+
y: [2, 4, 6, 8, 10],
448+
transforms: [{
449+
type: 'aggregate',
450+
groups: [1, 2, 2, 1, 1],
451+
aggregations: [
452+
{target: 'x', func: 'sum'},
453+
{target: 'y', func: 'avg'}
454+
]
455+
}, {
456+
type: 'filter',
457+
target: 'x',
458+
operation: '<',
459+
value: 5
460+
}]
461+
};
462+
463+
var trace2 = Lib.extendDeep({}, trace1);
464+
trace2.transforms.reverse();
465+
466+
Plotly.newPlot(gd, [trace1, trace2]);
467+
468+
var trace1Out = gd._fullData[0];
469+
expect(trace1Out.x).toEqual([2]);
470+
expect(trace1Out.y).toEqual([5]);
471+
472+
var trace2Out = gd._fullData[1];
473+
expect(trace2Out.x).toEqual([4, -5]);
474+
expect(trace2Out.y).toEqual([5, 4]);
475+
});
476+
477+
it('always executes groupby before aggregate', function() {
478+
// aggregate and groupby wouldn't commute, but groupby always happens first
479+
// because it has a `transform`, and aggregate has a `calcTransform`
480+
481+
var trace1 = {
482+
x: [1, 2, 3, 4, 5],
483+
y: [2, 4, 6, 8, 10],
484+
transforms: [{
485+
type: 'groupby',
486+
groups: [1, 1, 2, 2, 2]
487+
}, {
488+
type: 'aggregate',
489+
groups: [1, 2, 2, 1, 1],
490+
aggregations: [
491+
{target: 'x', func: 'sum'},
492+
{target: 'y', func: 'avg'}
493+
]
494+
}]
495+
};
496+
497+
var trace2 = Lib.extendDeep({}, trace1);
498+
trace2.transforms.reverse();
499+
500+
Plotly.newPlot(gd, [trace1, trace2]);
501+
502+
var t1g1 = gd._fullData[0];
503+
var t1g2 = gd._fullData[1];
504+
var t2g1 = gd._fullData[2];
505+
var t2g2 = gd._fullData[3];
506+
507+
expect(t1g1.x).toEqual([1, 2]);
508+
expect(t1g1.y).toEqual([2, 4]);
509+
// group 2 has its aggregations switched, since group 2 comes first
510+
expect(t1g2.x).toEqual([3, 9]);
511+
expect(t1g2.y).toEqual([6, 9]);
512+
513+
// if we had done aggregation first, we'd implicitly get the first val
514+
// for each of the groupby groups, which is [1, 1]
515+
// so we'd only make 1 output trace, and it would look like:
516+
// {x: [10, 5], y: [20/3, 5]}
517+
// (and if we got some other groupby groups values, the most it could do
518+
// is break ^^ into two separate traces)
519+
expect(t2g1.x).toEqual(t1g1.x);
520+
expect(t2g1.y).toEqual(t1g1.y);
521+
expect(t2g2.x).toEqual(t1g2.x);
522+
expect(t2g2.y).toEqual(t1g2.y);
523+
});
449524
});
450525

451526
describe('invalid transforms', function() {

0 commit comments

Comments
 (0)
Please sign in to comment.