From 130d72c872539c182e27524f9c77a1b73026bcc8 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 3 Nov 2015 18:35:01 -0500 Subject: [PATCH 1/6] adding a filterable json provider. --- lib/torque/provider/filterableJson.js | 663 ++++++++++++++++++++++++++ lib/torque/provider/index.js | 1 + 2 files changed, 664 insertions(+) create mode 100644 lib/torque/provider/filterableJson.js diff --git a/lib/torque/provider/filterableJson.js b/lib/torque/provider/filterableJson.js new file mode 100644 index 00000000..b94bc08b --- /dev/null +++ b/lib/torque/provider/filterableJson.js @@ -0,0 +1,663 @@ +var torque = require('../'); +var Profiler = require('../profiler'); + + var Uint8Array = torque.types.Uint8Array; + var Int32Array = torque.types.Int32Array; + var Uint32Array = torque.types.Uint32Array; + + // format('hello, {0}', 'rambo') -> "hello, rambo" + function format(str) { + for(var i = 1; i < arguments.length; ++i) { + var attrs = arguments[i]; + for(var attr in attrs) { + str = str.replace(RegExp('\\{' + attr + '\\}', 'g'), attrs[attr]); + } + } + return str; + } + + var filterableJson = function (options) { + this._ready = false; + this._tileQueue = []; + this.options = options; + this._filters = {}; + this._mapping = { + no:{ + type: 'float', + col_id:6 + }, + passenger_count:{ + type: 'float', + col_id: 0 + }, + tip_amount:{ + type:'float', + col_id:1 + }, + payment_type:{ + type:'cat', + col_ids:{ + "Credit card": 2, + "Cash": 3, + "No charge": 4, + "Dispute": 5 + } + }, + }; + + this.options.is_time = this.options.is_time === undefined ? true: this.options.is_time; + this.options.tiler_protocol = options.tiler_protocol || 'http'; + this.options.tiler_domain = options.tiler_domain || 'cartodb.com'; + this.options.tiler_port = options.tiler_port || 80; + + if (this.options.data_aggregation) { + this.options.cumulative = this.options.data_aggregation === 'cumulative'; + } + + // check options + if (options.resolution === undefined ) throw new Error("resolution should be provided"); + if (options.steps === undefined ) throw new Error("steps should be provided"); + if(options.start === undefined) { + this._fetchKeySpan(); + } else { + this._setReady(true); + } + }; + + + + filterableJson.prototype = { + + /** + * return the torque tile encoded in an efficient javascript + * structure: + * { + * x:Uint8Array x coordinates in tile reference system, normally from 0-255 + * y:Uint8Array y coordinates in tile reference system + * Index: Array index to the properties + * } + */ + proccessTile: function(rows, coord, zoom) { + var r; + var x = new Uint8Array(rows.length); + var y = new Uint8Array(rows.length); + + var prof_mem = Profiler.metric('ProviderJSON:mem'); + var prof_point_count = Profiler.metric('ProviderJSON:point_count'); + var prof_process_time = Profiler.metric('ProviderJSON:process_time').start() + + // count number of dates + var dates = 0; + var maxDateSlots = -1; + for (r = 0; r < rows.length; ++r) { + var row = rows[r]; + dates += row.dates__uint16.length; + for(var d = 0; d < row.dates__uint16.length; ++d) { + maxDateSlots = Math.max(maxDateSlots, row.dates__uint16[d]); + } + } + + if(this.options.cumulative) { + dates = (1 + maxDateSlots) * rows.length; + } + + var type = this.options.cumulative ? Uint32Array: Uint8Array; + + // reserve memory for all the dates + var timeIndex = new Int32Array(maxDateSlots + 1); //index-size + var timeCount = new Int32Array(maxDateSlots + 1); + var renderData = new (this.options.valueDataType || type)(dates); + var renderDataPos = new Uint32Array(dates); + + prof_mem.inc( + 4 * maxDateSlots + // timeIndex + 4 * maxDateSlots + // timeCount + dates + //renderData + dates * 4 + ); //renderDataPos + + prof_point_count.inc(rows.length); + + var rowsPerSlot = {}; + + // precache pixel positions + for (var r = 0; r < rows.length; ++r) { + var row = rows[r]; + x[r] = row.x__uint8 * this.options.resolution; + // fix value when it's in the tile EDGE + // TODO: this should be fixed in SQL query + if (row.y__uint8 === -1) { + y[r] = 0; + } else { + y[r] = row.y__uint8 * this.options.resolution; + } + + var dates = row.dates__uint16; + var vals = row.vals__uint8; + if (!this.options.cumulative) { + for (var j = 0, len = dates.length; j < len; ++j) { + var rr = rowsPerSlot[dates[j]] || (rowsPerSlot[dates[j]] = []); + if(this.options.cumulative) { + vals[j] += prev_val; + } + prev_val = vals[j]; + rr.push([r, vals[j]]); + } + } else { + var valByDate = {} + for (var j = 0, len = dates.length; j < len; ++j) { + valByDate[dates[j]] = vals[j]; + } + var accum = 0; + + // extend the latest to the end + for (var j = dates[0]; j <= maxDateSlots; ++j) { + var rr = rowsPerSlot[j] || (rowsPerSlot[j] = []); + var v = valByDate[j]; + if (v) { + accum += v; + } + rr.push([r, accum]); + } + + /*var lastDateSlot = dates[dates.length - 1]; + for (var j = lastDateSlot + 1; j <= maxDateSlots; ++j) { + var rr = rowsPerSlot[j] || (rowsPerSlot[j] = []); + rr.push([r, prev_val]); + } + */ + } + + } + + // for each timeslot search active buckets + var renderDataIndex = 0; + var timeSlotIndex = 0; + var i = 0; + for(var i = 0; i <= maxDateSlots; ++i) { + var c = 0; + var slotRows = rowsPerSlot[i] + if(slotRows) { + for (var r = 0; r < slotRows.length; ++r) { + var rr = slotRows[r]; + ++c; + renderDataPos[renderDataIndex] = rr[0] + renderData[renderDataIndex] = rr[1]; + ++renderDataIndex; + } + } + timeIndex[i] = timeSlotIndex; + timeCount[i] = c; + timeSlotIndex += c; + } + + prof_process_time.end(); + + return { + x: x, + y: y, + z: zoom, + coord: { + x: coord.x, + y: coord.y, + z: zoom + }, + timeCount: timeCount, + timeIndex: timeIndex, + renderDataPos: renderDataPos, + renderData: renderData, + maxDate: maxDateSlots + }; + }, + _generateFilterSQLForCat:function(name,categories){ + return name+" in "+categories; + }, + _generateFilterSQLForRange:function(name,range){ + var result = "" + if(range.start){ + result += " " + name + " > " + range.start; + } + if(range.end){ + result += " " + name + " < " + range.end; + } + return result + }, + _setFilters:function(filters){ + this.filters = filters + }, + _generateFiltersSQL:function(){ + return Object.keys(this._filters).map(function(filterName){ + var filter = this._filters[filterName] + if(filter){ + if(filter.type == 'range'){ + return this._generateFilterSQLForRange(filterName, filter.range) + } + else if(filter.type == 'cat'){ + return this._generateFilterSQLForCat(filterName, filter.categories) + } + else{ + return "" + } + } + else{ + return "" + } + }) + }, + _host: function() { + var opts = this.options; + var port = opts.sql_api_port; + var domain = ((opts.user_name || opts.user) + '.' + (opts.sql_api_domain || 'cartodb.com')) + (port ? ':' + port: ''); + var protocol = opts.sql_api_protocol || 'http'; + return this.options.url || protocol + '://' + domain + '/api/v2/sql'; + }, + + url: function(subhost) { + var opts = this.options; + var protocol = opts.sql_api_protocol || 'http'; + if (!this.options.cdn_url) { + return this._host(); + } + var h = protocol+ "://"; + if (subhost) { + h += subhost + "."; + } + var cdn_host = opts.cdn_url; + if(!cdn_host.http && !cdn_host.https) { + throw new Error("cdn_host should contain http and/or https entries"); + } + h += cdn_host[protocol] + "/" + (opts.user_name || opts.user) + '/api/v2/sql'; + return h; + }, + + _hash: function(str) { + var hash = 0; + if (!str || str.length == 0) return hash; + for (var i = 0, l = str.length; i < l; ++i) { + hash = (( (hash << 5 ) - hash ) + str.charCodeAt(i)) | 0; + } + return hash; + }, + + _extraParams: function() { + if (this.options.extra_params) { + var p = []; + for(var k in this.options.extra_params) { + var v = this.options.extra_params[k]; + if (v) { + p.push(k + "=" + encodeURIComponent(v)); + } + } + return p.join('&'); + } + return null; + }, + + isHttps: function() { + return this.options.sql_api_protocol && this.options.sql_api_protocol === 'https'; + }, + + // execute actual query + sql: function(sql, callback, options) { + options = options || {}; + var subdomains = this.options.subdomains || '0123'; + if(this.isHttps()) { + subdomains = [null]; // no subdomain + } + + + var url; + if (options.no_cdn) { + url = this._host(); + } else { + url = this.url(subdomains[Math.abs(this._hash(sql))%subdomains.length]); + } + var extra = this._extraParams(); + torque.net.get( url + "?q=" + encodeURIComponent(sql) + (extra ? "&" + extra: ''), function (data) { + if(options.parseJSON) { + data = JSON.parse(data && data.responseText); + } + callback && callback(data); + }); + }, + + getTileData: function(coord, zoom, callback) { + if(!this._ready) { + this._tileQueue.push([coord, zoom, callback]); + } else { + this._getTileData(coord, zoom, callback); + } + }, + + _setReady: function(ready) { + this._ready = true; + this._processQueue(); + this.options.ready && this.options.ready(); + }, + + _processQueue: function() { + var item; + while (item = this._tileQueue.pop()) { + this._getTileData.apply(this, item); + } + }, + + /** + * `coord` object like {x : tilex, y: tiley } + * `zoom` quadtree zoom level + */ + + _getTileData: function(coord, zoom, callback) { + var prof_fetch_time = Profiler.metric('ProviderJSON:tile_fetch_time').start() + this.table = this.options.table; + var numTiles = 1 << zoom; + + var column_conv = this.options.column; + + if(this.options.is_time) { + column_conv = format("date_part('epoch', {column})", this.options); + } + + var sql = "" + + "WITH " + + "par AS (" + + " SELECT CDB_XYZ_Resolution({zoom})*{resolution} as res" + + ", 256/{resolution} as tile_size" + + ", CDB_XYZ_Extent({x}, {y}, {zoom}) as ext " + + ")," + + "cte AS ( "+ + " SELECT ST_SnapToGrid(i.the_geom_webmercator, p.res) g" + + ", avg(passenger_count) c1, avg(tip_amount) c2, sum( case when payment_type=1 then 1 else 0 end) c3, sum(case when payment_type=2 then 1 else 0 end ) c4, sum(case when payment_type=3 then 1 else 0 end ) c5,sum(case when payment_type=4 then 1 else 0 end ) c6, count(cartodb_id) c7 " + + " FROM ({_sql}) i, par p " + + " WHERE i.the_geom_webmercator && p.ext " + + " GROUP BY g" + + ") " + + "" + + "SELECT (st_x(g)-st_xmin(p.ext))/p.res x__uint8, " + + " (st_y(g)-st_ymin(p.ext))/p.res y__uint8," + + " Array[c1,c2,c3,c4,c5,c6,c7] vals__uint8," + + " Array[0,1,2,3,4,5,6] dates__uint16" + + // the tile_size where are needed because the overlaps query in cte subquery includes the points + // in the left and bottom borders of the tile + " FROM cte, par p where (st_y(g)-st_ymin(p.ext))/p.res < tile_size and (st_x(g)-st_xmin(p.ext))/p.res < tile_size "; + + + var query = format(sql, this.options, { + zoom: zoom, + x: coord.x, + y: coord.y, + column_conv: column_conv, + _sql: this.getSQL() + }); + + + var self = this; + this.sql(query, function (data) { + if (data) { + var rows = JSON.parse(data.responseText).rows; + callback(self.proccessTile(rows, coord, zoom)); + } else { + callback(null); + } + prof_fetch_time.end(); + }); + }, + + getKeySpan: function() { + return { + start: this.options.start * 1000, + end: this.options.end * 1000, + step: this.options.step, + steps: this.options.steps, + columnType: this.options.is_time ? 'date': 'number' + }; + }, + + setColumn: function(column, isTime) { + this.options.column = column; + this.options.is_time = isTime === undefined ? true: false; + this.reload(); + }, + + setResolution: function(res) { + this.options.resolution = res; + }, + + idForRange: function(variable){ + return this._mapping[variable].col_id + }, + idsForCategory:function(variable){ + return this._mapping[variable].col_ids + }, + idForCategoryOption:function(variable, option){ + return this._mapping[variable].col_ids.to_a.select(function(a){return option==a[0]})[0][1] + }, + optionForId:function(id){ + Object.keys(this._mapping).forEach(function(varaibleName){ + var variable = this._mapping[variableName] + if(variable.type=='range' && variable.col_id==id){ + return variableName + } + else if(variable.type=='cat') { + Object.keys(variable.col_ids).forEach(function(key){ + if(variable.col_ids[key] == id){ + + } + }.bind(this)) + } + }.bind(this)) + return null; + }, + // return true if tiles has been changed + setOptions: function(opt) { + var refresh = false; + + if(opt.resolution !== undefined && opt.resolution !== this.options.resolution) { + this.options.resolution = opt.resolution; + refresh = true; + } + + if(opt.steps !== undefined && opt.steps !== this.options.steps) { + this.setSteps(opt.steps, { silent: true }); + refresh = true; + } + + if(opt.column !== undefined && opt.column !== this.options.column) { + this.options.column = opt.column; + refresh = true; + } + + if(opt.countby !== undefined && opt.countby !== this.options.countby) { + this.options.countby = opt.countby; + refresh = true; + } + + if(opt.data_aggregation !== undefined) { + var c = opt.data_aggregation === 'cumulative'; + if (this.options.cumulative !== c) { + this.options.cumulative = c; + refresh = true; + } + } + + if (refresh) this.reload(); + return refresh; + + }, + + reload: function() { + this._ready = false; + this._fetchKeySpan(); + }, + + setSQL: function(sql) { + if (this.options.sql != sql) { + this.options.sql = sql; + this.reload(); + } + }, + + getSteps: function() { + return Math.min(this.options.steps, this.options.data_steps); + }, + + setSteps: function(steps, opt) { + opt = opt || {}; + if (this.options.steps !== steps) { + this.options.steps = steps; + this.options.step = (this.options.end - this.options.start)/this.getSteps(); + this.options.step = this.options.step || 1; + if (!opt.silent) this.reload(); + } + }, + + getBounds: function() { + return this.options.bounds; + }, + + getSQL: function() { + return this.options.sql || "select * from " + this.options.table; + }, + + _tilerHost: function() { + var opts = this.options; + var user = (opts.user_name || opts.user); + return opts.tiler_protocol + + "://" + (user ? user + "." : "") + + opts.tiler_domain + + ((opts.tiler_port != "") ? (":" + opts.tiler_port) : ""); + }, + + _fetchUpdateAt: function(callback) { + var self = this; + var layergroup = { + "version": "1.0.1", + "stat_tag": this.options.stat_tag || 'torque', + "layers": [{ + "type": "cartodb", + "options": { + "cartocss_version": "2.1.1", + "cartocss": "#layer {}", + "sql": this.getSQL() + } + }] + }; + var url = this._tilerHost() + "/tiles/layergroup"; + var extra = this._extraParams(); + + // tiler needs map_key instead of api_key + // so replace it + if (extra) { + extra = extra.replace('api_key=', 'map_key='); + } + + url = url + + "?config=" + encodeURIComponent(JSON.stringify(layergroup)) + + "&callback=?" + (extra ? "&" + extra: ''); + + torque.net.jsonp(url, function (data) { + var query = format("select * from ({sql}) __torque_wrap_sql limit 0", { sql: self.getSQL() }); + self.sql(query, function (queryData) { + if (data && queryData) { + callback({ + updated_at: data.last_updated, + fields: queryData.fields + }); + } + }, { parseJSON: true }); + }); + }, + + // + // the data range could be set by the user though ``start`` + // option. It can be fecthed from the table when the start + // is not specified. + // + _fetchKeySpan: function() { + var self = this; + var max_col, min_col, max_tmpl, min_tmpl; + + this._fetchUpdateAt(function(data) { + if (!data) return; + self.options.extra_params = self.options.extra_params || {}; + self.options.extra_params.last_updated = data.updated_at || 0; + self.options.extra_params.cache_policy = 'persist'; + self.options.is_time = data.fields[self.options.column].type === 'date'; + + var column_conv = self.options.column; + if (self.options.is_time){ + max_tmpl = "date_part('epoch', max({column}))"; + min_tmpl = "date_part('epoch', min({column}))"; + column_conv = format("date_part('epoch', {column})", self.options); + } else { + max_tmpl = "max({column})"; + min_tmpl = "min({column})"; + } + + max_col = format(max_tmpl, { column: self.options.column }); + min_col = format(min_tmpl, { column: self.options.column }); + + /*var sql_stats = "" + + "WITH summary_groups as ( " + + "WITH summary as ( " + + "select (row_number() over (order by __time_col asc nulls last)+1)/2 as rownum, __time_col " + + "from (select *, {column} as __time_col from ({sql}) __s) __torque_wrap_sql " + + "order by __time_col asc " + + ") " + + "SELECT " + + "max(__time_col) OVER(PARTITION BY rownum) - " + + "min(__time_col) OVER(PARTITION BY rownum) diff " + + "FROM summary " + + "), subq as ( " + + " SELECT " + + "st_xmax(st_envelope(st_collect(the_geom))) xmax, " + + "st_ymax(st_envelope(st_collect(the_geom))) ymax, " + + "st_xmin(st_envelope(st_collect(the_geom))) xmin, " + + "st_ymin(st_envelope(st_collect(the_geom))) ymin, " + + "{max_col} max, " + + "{min_col} min FROM ({sql}) __torque_wrap_sql " + + ")" + + "SELECT " + + "xmax, xmin, ymax, ymin, a.max as max_date, a.min as min_date, " + + "avg(diff) as diffavg," + + "(a.max - a.min)/avg(diff) as num_steps " + + "FROM summary_groups, subq a " + + "WHERE diff > 0 group by xmax, xmin, ymax, ymin, max_date, min_date"; + */ + var sql_stats = " SELECT " + + "st_xmax(st_envelope(st_collect(the_geom))) xmax, " + + "st_ymax(st_envelope(st_collect(the_geom))) ymax, " + + "st_xmin(st_envelope(st_collect(the_geom))) xmin, " + + "st_ymin(st_envelope(st_collect(the_geom))) ymin, " + + "count(*) as num_steps, " + + "{max_col} max_date, " + + "{min_col} min_date FROM ({sql}) __torque_wrap_sql "; + + var sql = format(sql_stats, { + max_col: max_col, + min_col: min_col, + column: column_conv, + sql: self.getSQL() + }); + + self.sql(sql, function(data) { + //TODO: manage bounds + data = data.rows[0]; + self.options.start = data.min_date; + self.options.end = data.max_date; + self.options.step = (data.max_date - data.min_date)/Math.min(self.options.steps, data.num_steps>>0); + self.options.data_steps = data.num_steps >> 0; + // step can't be 0 + self.options.step = self.options.step || 1; + self.options.bounds = [ + [data.ymin, data.xmin], + [data.ymax, data.xmax] + ]; + self._setReady(true); + }, { parseJSON: true, no_cdn: true }); + }, { parseJSON: true, no_cdn: true}) + } + + }; + +module.exports = filterableJson; diff --git a/lib/torque/provider/index.js b/lib/torque/provider/index.js index 7073e03a..43c2848e 100644 --- a/lib/torque/provider/index.js +++ b/lib/torque/provider/index.js @@ -1,5 +1,6 @@ module.exports = { json: require('./json'), + filterableJson: require('./filterableJson'), JsonArray: require('./jsonarray'), windshaft: require('./windshaft'), tileJSON: require('./tilejson') From fd5bc0f732636a28081f47b07cbdb10918795d14 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 3 Nov 2015 18:35:56 -0500 Subject: [PATCH 2/6] Adding the ability for leafletLayer to filter data on the client side and to calculate histograms and value arrays for the variables stored in the currently loaded tiles. --- lib/torque/leaflet/torque.js | 159 ++++++++++++++++++++++++++++++++++- lib/torque/renderer/point.js | 45 +++++----- 2 files changed, 181 insertions(+), 23 deletions(-) diff --git a/lib/torque/leaflet/torque.js b/lib/torque/leaflet/torque.js index 1c267194..ab7d98fa 100644 --- a/lib/torque/leaflet/torque.js +++ b/lib/torque/leaflet/torque.js @@ -10,6 +10,7 @@ L.TorqueLayer = L.CanvasLayer.extend({ providers: { 'sql_api': torque.providers.json, + 'filterable_sql_api': torque.providers.filterableJson, 'url_template': torque.providers.JsonArray, 'windshaft': torque.providers.windshaft, 'tileJSON': torque.providers.tileJSON @@ -28,6 +29,7 @@ L.TorqueLayer = L.CanvasLayer.extend({ options.tileLoader = true; this.key = 0; this.prevRenderedKey = 0; + this._filters = {} if (options.cartocss) { torque.extend(options, torque.common.TorqueLayer.optionsFromCartoCSS(options.cartocss)); } @@ -98,11 +100,42 @@ L.TorqueLayer = L.CanvasLayer.extend({ self.redraw(); } self.fire('tileLoaded'); + self.fire('dataUpdate') }); }, this); }, + setFilters:function(){ + this.provider.setFilters(this._filters); + this._reloadTiles(); + return this; + }, + filterByRange:function(variableName,start,end){ + this._filters[variableName] = {type: 'range', range: {start:start,end:end}} + this._filtersChanged() + this.fire('dataUpdate') + return this + }, + filterByCat:function(variableName,categories){ + this._filters[variableName] = {type: 'cat', categories: categories} + this._filtersChanged() + return this + }, + clearFilter: function(name){ + if(name){ + delete this._filters[name] + } + else{ + this._filters = {} + } + this._filtersChanged() + return this + }, + _filtersChanged:function(){ + this._clearTileCaches() + this._render() + }, _clearTileCaches: function() { var t, tile; for(t in this._tiles) { @@ -112,12 +145,101 @@ L.TorqueLayer = L.CanvasLayer.extend({ } } }, - _clearCaches: function() { this.renderer && this.renderer.clearSpriteCache(); this._clearTileCaches(); }, + valuesForRangeVariable:function(variable){ + var t, tile; + + var variable_id = this.provider.idForRange(variable) + var values = [ ] + + + for(t in this._tiles){ + tile = this._tiles[t] + var noPoints = tile.x.length; + for(var i=0; i < tile.x.length; i++){ + if(tile.renderFlags[i] ){ + value = tile.renderData[variable_id*noPoints + i] + values.push(value) + } + } + } + return values; + }, + + valuesForCatVariable:function(variable){ + var t, tile; + + var categories = this.provider.idsForCategory(variable) + var result = [ ] + + + for(t in this._tiles){ + tile = this._tiles[t] + var noPoints = tile.x.length; + for(var i=0; i < tile.x.length; i++){ + if(tile.renderFlags[i] ){ + var vals={} + Object.keys(categories).forEach(function(categoryName){ + var variable_id = categories[categoryName] + value = tile.renderData[variable_id*noPoints + i] + vals[categoryName] = value + }.bind(this)) + result.push(vals) + } + } + } + return result; + + }, + getValues:function(variable,callback){ + var type= this.provider._mapping[variable].type + if(type=='float'){ + callback(this.valuesForRangeVariable(variable)) + } + else{ + callback(this.valuesForCatVariable(variable)) + } + }, + getHistogram:function(variable,callback,noBins){ + var type= this.provider._mapping[variable].type + if(type=='float'){ + callback(this.histogramForRangeVariable(variable,noBins)) + } + else if(type='cat'){ + callback(this.histogramForCatVariable(variable)) + } + return this + }, + histogramForCatVariable:function(variable){ + var result = {} + this.valuesForCatVariable(variable).forEach(function(point){ + Object.keys(point).forEach(function(key){ + result[key] = result[key] || 0 + result[key] += point[key] + }) + }) + return result + }, + histogramForRangeVariable:function(variable,noBins){ + noBins = noBins || 10 + + var vals = this.valuesForRangeVariable(variable) + + var min = Math.min.apply(null, vals) + var max = Math.max.apply(null, vals) + var binSize = (max-min)/noBins + var result = [] + vals.forEach(function(val){ + var bin = (val -min)/binSize + result[bin]= result[bin] || 0 + result[bin] += val + }) + return result + }, onAdd: function (map) { map.on({ 'zoomend': this._clearCaches, @@ -208,6 +330,38 @@ L.TorqueLayer = L.CanvasLayer.extend({ canvas.width = canvas.width; }, + _filterTile:function(tile){ + var noPoints = tile.x.length + + var renderFlags = [] + for(var i =0; i < noPoints; i++){ + var includePoint = true + Object.keys(this._filters).forEach(function(key){ + var filter = this._filters[key] + var variableId = this.provider.idForRange(key) + + var value = tile.renderData[variableId*noPoints+i] + if(filter.type=='range'){ + if(value < filter.range.start || value > filter.range.end){ + includePoint = false; + } + } + else if (filter.type=='cat'){ + var ids = this.provider.idsForCategory(key); + filter.categories.forEach(function(key){ + var catId = ids[key] + var value = tile.renderData[catId*noPoints + i] + if(value==0){ + includePoint= false + } + }.bind(this)) + } + }.bind(this)) + renderFlags[i] = includePoint + } + return renderFlags + }, + /** * render the selectef key * don't call this function directly, it's called by @@ -236,10 +390,13 @@ L.TorqueLayer = L.CanvasLayer.extend({ // all the points this.renderer._ctx.drawImage(tile._tileCache, 0, 0); } else { + + tile.renderFlags=this._filterTile(tile) this.renderer.renderTile(tile, this.key); } } } + this.renderer.applyFilters(); // prepare caches if the animation is not running diff --git a/lib/torque/renderer/point.js b/lib/torque/renderer/point.js index 4b07d149..6153e15e 100644 --- a/lib/torque/renderer/point.js +++ b/lib/torque/renderer/point.js @@ -57,7 +57,7 @@ var Filters = require('./torque_filters'); this.TILE_SIZE = 256; this._style = null; this._gradients = {}; - + this._forcePoints = false; } @@ -165,14 +165,14 @@ var Filters = require('./torque_filters'); i.src = canvas.toDataURL(); return i; } - + return canvas; }, // // renders all the layers (and frames for each layer) from cartocss // - renderTile: function(tile, key, callback) { + renderTile: function(tile, key, renderFlags, callback) { if (this._iconsToLoad > 0) { this.on('allIconsLoaded', function() { this.renderTile.apply(this, [tile, key, callback]); @@ -193,7 +193,7 @@ var Filters = require('./torque_filters'); } } } - + prof.end(true); return callback && callback(null); @@ -237,12 +237,11 @@ var Filters = require('./torque_filters'); }, // - // renders a tile in the canvas for key defined in + // renders a tile in the canvas for key defined in // the torque tile // _renderTile: function(tile, key, frame_offset, sprites, shader, shaderVars) { if (!this._canvas) return; - var prof = Profiler.metric('torque.renderer.point.renderTile').start(); var ctx = this._ctx; var blendMode = compop2canvas(shader.eval('comp-op')) || this.options.blendmode; @@ -254,27 +253,29 @@ var Filters = require('./torque_filters'); key = tile.maxDate; } var tileMax = this.options.resolution * (this.TILE_SIZE/this.options.resolution - 1) - var activePixels = tile.timeCount[key]; + var activePixels = tile.x.length; var anchor = this.options.resolution/2; if (activePixels) { - var pixelIndex = tile.timeIndex[key]; + var pixelIndex = 0;//tile.timeIndex[key]; for(var p = 0; p < activePixels; ++p) { - var posIdx = tile.renderDataPos[pixelIndex + p]; - var c = tile.renderData[pixelIndex + p]; - if (c) { - var sp = sprites[c]; - if (sp === undefined) { - sp = sprites[c] = this.generateSprite(shader, c, torque.extend({ zoom: tile.z, 'frame-offset': frame_offset }, shaderVars)); - } - if (sp) { - var x = tile.x[posIdx]- (sp.width >> 1) + anchor; - var y = tileMax - tile.y[posIdx] + anchor; // flip mercator - ctx.drawImage(sp, x, y - (sp.height >> 1)); - } + if(tile.renderFlags[p]){ + var posIdx = tile.renderDataPos[pixelIndex + p]; + var c = tile.renderData[pixelIndex + p]; + if (c) { + var sp = sprites[c]; + if (sp === undefined) { + sp = sprites[c] = this.generateSprite(shader, c, torque.extend({ zoom: tile.z, 'frame-offset': frame_offset }, shaderVars)); + } + if (sp) { + var x = tile.x[posIdx]- (sp.width >> 1) + anchor; + var y = tileMax - tile.y[posIdx] + anchor; // flip mercator + ctx.drawImage(sp, x, y - (sp.height >> 1)); + } + } } } } - + prof.end(true); }, @@ -442,7 +443,7 @@ var Filters = require('./torque_filters'); } gradient = {}; var colorize = this._style['image-filters'].args; - + var increment = 1/colorize.length; for (var i = 0; i < colorize.length; i++){ var key = increment * i + increment; From 71fc89a8d89cb166bd88a49d14c6e5571527e246 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 3 Nov 2015 18:36:10 -0500 Subject: [PATCH 3/6] Examples of the filters and histogram api working --- examples/bi.html | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 examples/bi.html diff --git a/examples/bi.html b/examples/bi.html new file mode 100644 index 00000000..6ddb0a57 --- /dev/null +++ b/examples/bi.html @@ -0,0 +1,88 @@ + + + + + +
+
+ + + + + + + + + From 4ad0dba547e2a57300064c484d42e196eb846356 Mon Sep 17 00:00:00 2001 From: javi Date: Fri, 6 Nov 2015 17:35:49 +0100 Subject: [PATCH 4/6] removed client side filters temporally, integrated with server side queries --- lib/torque/leaflet/torque.js | 23 ++- lib/torque/provider/filterableJson.js | 258 +++++--------------------- lib/torque/renderer/point.js | 12 +- 3 files changed, 66 insertions(+), 227 deletions(-) diff --git a/lib/torque/leaflet/torque.js b/lib/torque/leaflet/torque.js index ab7d98fa..492f21d1 100644 --- a/lib/torque/leaflet/torque.js +++ b/lib/torque/leaflet/torque.js @@ -106,36 +106,42 @@ L.TorqueLayer = L.CanvasLayer.extend({ }, - setFilters:function(){ + setFilters: function() { this.provider.setFilters(this._filters); this._reloadTiles(); return this; }, - filterByRange:function(variableName,start,end){ - this._filters[variableName] = {type: 'range', range: {start:start,end:end}} + + filterByRange: function(variableName, start, end) { + this._filters[variableName] = {type: 'range', range: {start: start, end: end} } this._filtersChanged() this.fire('dataUpdate') return this }, - filterByCat:function(variableName,categories){ + + filterByCat: function(variableName, categories) { this._filters[variableName] = {type: 'cat', categories: categories} this._filtersChanged() return this }, + clearFilter: function(name){ - if(name){ + if(name) { delete this._filters[name] } - else{ + else { this._filters = {} } this._filtersChanged() return this }, + _filtersChanged:function(){ + this.provider._filters = this._filters; this._clearTileCaches() this._render() }, + _clearTileCaches: function() { var t, tile; for(t in this._tiles) { @@ -145,6 +151,7 @@ L.TorqueLayer = L.CanvasLayer.extend({ } } }, + _clearCaches: function() { this.renderer && this.renderer.clearSpriteCache(); this._clearTileCaches(); @@ -330,6 +337,7 @@ L.TorqueLayer = L.CanvasLayer.extend({ canvas.width = canvas.width; }, + /* _filterTile:function(tile){ var noPoints = tile.x.length @@ -361,6 +369,7 @@ L.TorqueLayer = L.CanvasLayer.extend({ } return renderFlags }, + */ /** * render the selectef key @@ -391,7 +400,7 @@ L.TorqueLayer = L.CanvasLayer.extend({ this.renderer._ctx.drawImage(tile._tileCache, 0, 0); } else { - tile.renderFlags=this._filterTile(tile) + //tile.renderFlags = this._filterTile(tile) this.renderer.renderTile(tile, this.key); } } diff --git a/lib/torque/provider/filterableJson.js b/lib/torque/provider/filterableJson.js index b94bc08b..8cd9aab4 100644 --- a/lib/torque/provider/filterableJson.js +++ b/lib/torque/provider/filterableJson.js @@ -86,34 +86,34 @@ var Profiler = require('../profiler'); var prof_point_count = Profiler.metric('ProviderJSON:point_count'); var prof_process_time = Profiler.metric('ProviderJSON:process_time').start() - // count number of dates - var dates = 0; + // count number of steps + var steps = 0; var maxDateSlots = -1; for (r = 0; r < rows.length; ++r) { var row = rows[r]; - dates += row.dates__uint16.length; - for(var d = 0; d < row.dates__uint16.length; ++d) { - maxDateSlots = Math.max(maxDateSlots, row.dates__uint16[d]); + steps += row.steps.length; + for(var d = 0; d < row.steps.length; ++d) { + maxDateSlots = Math.max(maxDateSlots, row.steps[d]); } } if(this.options.cumulative) { - dates = (1 + maxDateSlots) * rows.length; + steps = (1 + maxDateSlots) * rows.length; } var type = this.options.cumulative ? Uint32Array: Uint8Array; - // reserve memory for all the dates + // reserve memory for all the steps var timeIndex = new Int32Array(maxDateSlots + 1); //index-size var timeCount = new Int32Array(maxDateSlots + 1); - var renderData = new (this.options.valueDataType || type)(dates); - var renderDataPos = new Uint32Array(dates); + var renderData = new (this.options.valueDataType || type)(steps); + var renderDataPos = new Uint32Array(steps); prof_mem.inc( 4 * maxDateSlots + // timeIndex 4 * maxDateSlots + // timeCount - dates + //renderData - dates * 4 + steps + //renderData + steps * 4 ); //renderDataPos prof_point_count.inc(rows.length); @@ -123,20 +123,20 @@ var Profiler = require('../profiler'); // precache pixel positions for (var r = 0; r < rows.length; ++r) { var row = rows[r]; - x[r] = row.x__uint8 * this.options.resolution; + x[r] = row.x * this.options.resolution; // fix value when it's in the tile EDGE // TODO: this should be fixed in SQL query - if (row.y__uint8 === -1) { + if (row.y === -1) { y[r] = 0; } else { - y[r] = row.y__uint8 * this.options.resolution; + y[r] = row.y * this.options.resolution; } - var dates = row.dates__uint16; - var vals = row.vals__uint8; + var steps = row.steps; + var vals = row.vals; if (!this.options.cumulative) { - for (var j = 0, len = dates.length; j < len; ++j) { - var rr = rowsPerSlot[dates[j]] || (rowsPerSlot[dates[j]] = []); + for (var j = 0, len = steps.length; j < len; ++j) { + var rr = rowsPerSlot[steps[j]] || (rowsPerSlot[steps[j]] = []); if(this.options.cumulative) { vals[j] += prev_val; } @@ -145,13 +145,13 @@ var Profiler = require('../profiler'); } } else { var valByDate = {} - for (var j = 0, len = dates.length; j < len; ++j) { - valByDate[dates[j]] = vals[j]; + for (var j = 0, len = steps.length; j < len; ++j) { + valByDate[steps[j]] = vals[j]; } var accum = 0; // extend the latest to the end - for (var j = dates[0]; j <= maxDateSlots; ++j) { + for (var j = steps[0]; j <= maxDateSlots; ++j) { var rr = rowsPerSlot[j] || (rowsPerSlot[j] = []); var v = valByDate[j]; if (v) { @@ -160,7 +160,7 @@ var Profiler = require('../profiler'); rr.push([r, accum]); } - /*var lastDateSlot = dates[dates.length - 1]; + /*var lastDateSlot = steps[steps.length - 1]; for (var j = lastDateSlot + 1; j <= maxDateSlots; ++j) { var rr = rowsPerSlot[j] || (rowsPerSlot[j] = []); rr.push([r, prev_val]); @@ -209,33 +209,41 @@ var Profiler = require('../profiler'); maxDate: maxDateSlots }; }, + _generateFilterSQLForCat:function(name,categories){ return name+" in "+categories; }, + _generateFilterSQLForRange:function(name,range){ var result = "" - if(range.start){ + if (range.start) { result += " " + name + " > " + range.start; } - if(range.end){ + if (range.end) { + if (range.start) { + result += " and " + } result += " " + name + " < " + range.end; } return result }, + _setFilters:function(filters){ this.filters = filters }, - _generateFiltersSQL:function(){ + + _generateFiltersSQL: function() { + var self = this; return Object.keys(this._filters).map(function(filterName){ - var filter = this._filters[filterName] - if(filter){ - if(filter.type == 'range'){ - return this._generateFilterSQLForRange(filterName, filter.range) + var filter = self._filters[filterName] + if (filter) { + if (filter.type == 'range') { + return self._generateFilterSQLForRange(filterName, filter.range) } - else if(filter.type == 'cat'){ - return this._generateFilterSQLForCat(filterName, filter.categories) + else if (filter.type == 'cat') { + return self._generateFilterSQLForCat(filterName, filter.categories) } - else{ + else { return "" } } @@ -244,6 +252,7 @@ var Profiler = require('../profiler'); } }) }, + _host: function() { var opts = this.options; var port = opts.sql_api_port; @@ -354,40 +363,16 @@ var Profiler = require('../profiler'); var column_conv = this.options.column; - if(this.options.is_time) { - column_conv = format("date_part('epoch', {column})", this.options); - } - - var sql = "" + - "WITH " + - "par AS (" + - " SELECT CDB_XYZ_Resolution({zoom})*{resolution} as res" + - ", 256/{resolution} as tile_size" + - ", CDB_XYZ_Extent({x}, {y}, {zoom}) as ext " + - ")," + - "cte AS ( "+ - " SELECT ST_SnapToGrid(i.the_geom_webmercator, p.res) g" + - ", avg(passenger_count) c1, avg(tip_amount) c2, sum( case when payment_type=1 then 1 else 0 end) c3, sum(case when payment_type=2 then 1 else 0 end ) c4, sum(case when payment_type=3 then 1 else 0 end ) c5,sum(case when payment_type=4 then 1 else 0 end ) c6, count(cartodb_id) c7 " + - " FROM ({_sql}) i, par p " + - " WHERE i.the_geom_webmercator && p.ext " + - " GROUP BY g" + - ") " + - "" + - "SELECT (st_x(g)-st_xmin(p.ext))/p.res x__uint8, " + - " (st_y(g)-st_ymin(p.ext))/p.res y__uint8," + - " Array[c1,c2,c3,c4,c5,c6,c7] vals__uint8," + - " Array[0,1,2,3,4,5,6] dates__uint16" + - // the tile_size where are needed because the overlaps query in cte subquery includes the points - // in the left and bottom borders of the tile - " FROM cte, par p where (st_y(g)-st_ymin(p.ext))/p.res < tile_size and (st_x(g)-st_xmin(p.ext))/p.res < tile_size "; + var sql = "select * from torque_tile({x}, {y}, {zoom}, ARRAY['avg({column})'], '{table}', '{filters}')"; var query = format(sql, this.options, { zoom: zoom, x: coord.x, y: coord.y, - column_conv: column_conv, - _sql: this.getSQL() + column: column_conv, + table: this.options.table, + filters: this._generateFiltersSQL() }); @@ -423,31 +408,6 @@ var Profiler = require('../profiler'); this.options.resolution = res; }, - idForRange: function(variable){ - return this._mapping[variable].col_id - }, - idsForCategory:function(variable){ - return this._mapping[variable].col_ids - }, - idForCategoryOption:function(variable, option){ - return this._mapping[variable].col_ids.to_a.select(function(a){return option==a[0]})[0][1] - }, - optionForId:function(id){ - Object.keys(this._mapping).forEach(function(varaibleName){ - var variable = this._mapping[variableName] - if(variable.type=='range' && variable.col_id==id){ - return variableName - } - else if(variable.type=='cat') { - Object.keys(variable.col_ids).forEach(function(key){ - if(variable.col_ids[key] == id){ - - } - }.bind(this)) - } - }.bind(this)) - return null; - }, // return true if tiles has been changed setOptions: function(opt) { var refresh = false; @@ -528,134 +488,8 @@ var Profiler = require('../profiler'); ((opts.tiler_port != "") ? (":" + opts.tiler_port) : ""); }, - _fetchUpdateAt: function(callback) { - var self = this; - var layergroup = { - "version": "1.0.1", - "stat_tag": this.options.stat_tag || 'torque', - "layers": [{ - "type": "cartodb", - "options": { - "cartocss_version": "2.1.1", - "cartocss": "#layer {}", - "sql": this.getSQL() - } - }] - }; - var url = this._tilerHost() + "/tiles/layergroup"; - var extra = this._extraParams(); - - // tiler needs map_key instead of api_key - // so replace it - if (extra) { - extra = extra.replace('api_key=', 'map_key='); - } - - url = url + - "?config=" + encodeURIComponent(JSON.stringify(layergroup)) + - "&callback=?" + (extra ? "&" + extra: ''); - - torque.net.jsonp(url, function (data) { - var query = format("select * from ({sql}) __torque_wrap_sql limit 0", { sql: self.getSQL() }); - self.sql(query, function (queryData) { - if (data && queryData) { - callback({ - updated_at: data.last_updated, - fields: queryData.fields - }); - } - }, { parseJSON: true }); - }); - }, - - // - // the data range could be set by the user though ``start`` - // option. It can be fecthed from the table when the start - // is not specified. - // _fetchKeySpan: function() { - var self = this; - var max_col, min_col, max_tmpl, min_tmpl; - - this._fetchUpdateAt(function(data) { - if (!data) return; - self.options.extra_params = self.options.extra_params || {}; - self.options.extra_params.last_updated = data.updated_at || 0; - self.options.extra_params.cache_policy = 'persist'; - self.options.is_time = data.fields[self.options.column].type === 'date'; - - var column_conv = self.options.column; - if (self.options.is_time){ - max_tmpl = "date_part('epoch', max({column}))"; - min_tmpl = "date_part('epoch', min({column}))"; - column_conv = format("date_part('epoch', {column})", self.options); - } else { - max_tmpl = "max({column})"; - min_tmpl = "min({column})"; - } - - max_col = format(max_tmpl, { column: self.options.column }); - min_col = format(min_tmpl, { column: self.options.column }); - - /*var sql_stats = "" + - "WITH summary_groups as ( " + - "WITH summary as ( " + - "select (row_number() over (order by __time_col asc nulls last)+1)/2 as rownum, __time_col " + - "from (select *, {column} as __time_col from ({sql}) __s) __torque_wrap_sql " + - "order by __time_col asc " + - ") " + - "SELECT " + - "max(__time_col) OVER(PARTITION BY rownum) - " + - "min(__time_col) OVER(PARTITION BY rownum) diff " + - "FROM summary " + - "), subq as ( " + - " SELECT " + - "st_xmax(st_envelope(st_collect(the_geom))) xmax, " + - "st_ymax(st_envelope(st_collect(the_geom))) ymax, " + - "st_xmin(st_envelope(st_collect(the_geom))) xmin, " + - "st_ymin(st_envelope(st_collect(the_geom))) ymin, " + - "{max_col} max, " + - "{min_col} min FROM ({sql}) __torque_wrap_sql " + - ")" + - "SELECT " + - "xmax, xmin, ymax, ymin, a.max as max_date, a.min as min_date, " + - "avg(diff) as diffavg," + - "(a.max - a.min)/avg(diff) as num_steps " + - "FROM summary_groups, subq a " + - "WHERE diff > 0 group by xmax, xmin, ymax, ymin, max_date, min_date"; - */ - var sql_stats = " SELECT " + - "st_xmax(st_envelope(st_collect(the_geom))) xmax, " + - "st_ymax(st_envelope(st_collect(the_geom))) ymax, " + - "st_xmin(st_envelope(st_collect(the_geom))) xmin, " + - "st_ymin(st_envelope(st_collect(the_geom))) ymin, " + - "count(*) as num_steps, " + - "{max_col} max_date, " + - "{min_col} min_date FROM ({sql}) __torque_wrap_sql "; - - var sql = format(sql_stats, { - max_col: max_col, - min_col: min_col, - column: column_conv, - sql: self.getSQL() - }); - - self.sql(sql, function(data) { - //TODO: manage bounds - data = data.rows[0]; - self.options.start = data.min_date; - self.options.end = data.max_date; - self.options.step = (data.max_date - data.min_date)/Math.min(self.options.steps, data.num_steps>>0); - self.options.data_steps = data.num_steps >> 0; - // step can't be 0 - self.options.step = self.options.step || 1; - self.options.bounds = [ - [data.ymin, data.xmin], - [data.ymax, data.xmax] - ]; - self._setReady(true); - }, { parseJSON: true, no_cdn: true }); - }, { parseJSON: true, no_cdn: true}) + this._setReady(true); } }; diff --git a/lib/torque/renderer/point.js b/lib/torque/renderer/point.js index 6153e15e..da6d687d 100644 --- a/lib/torque/renderer/point.js +++ b/lib/torque/renderer/point.js @@ -258,21 +258,17 @@ var Filters = require('./torque_filters'); if (activePixels) { var pixelIndex = 0;//tile.timeIndex[key]; for(var p = 0; p < activePixels; ++p) { - if(tile.renderFlags[p]){ var posIdx = tile.renderDataPos[pixelIndex + p]; var c = tile.renderData[pixelIndex + p]; - if (c) { - var sp = sprites[c]; - if (sp === undefined) { + var sp = sprites[c]; + if (sp === undefined) { sp = sprites[c] = this.generateSprite(shader, c, torque.extend({ zoom: tile.z, 'frame-offset': frame_offset }, shaderVars)); - } - if (sp) { + } + if (sp) { var x = tile.x[posIdx]- (sp.width >> 1) + anchor; var y = tileMax - tile.y[posIdx] + anchor; // flip mercator ctx.drawImage(sp, x, y - (sp.height >> 1)); - } } - } } } From dea2f1f27f2576a98254bf60479bfdebfd566cc0 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 6 Nov 2015 17:06:21 +0000 Subject: [PATCH 5/6] updating code to use the new tile format --- lib/torque/provider/filterableJson.js | 141 +++++++++----------------- lib/torque/renderer/point.js | 3 +- 2 files changed, 51 insertions(+), 93 deletions(-) diff --git a/lib/torque/provider/filterableJson.js b/lib/torque/provider/filterableJson.js index b94bc08b..3e5cb70a 100644 --- a/lib/torque/provider/filterableJson.js +++ b/lib/torque/provider/filterableJson.js @@ -91,23 +91,21 @@ var Profiler = require('../profiler'); var maxDateSlots = -1; for (r = 0; r < rows.length; ++r) { var row = rows[r]; - dates += row.dates__uint16.length; - for(var d = 0; d < row.dates__uint16.length; ++d) { - maxDateSlots = Math.max(maxDateSlots, row.dates__uint16[d]); + dates += row.steps.length; + for(var d = 0; d < row.steps.length; ++d) { + maxDateSlots = Math.max(maxDateSlots, row.steps[d]); } } - if(this.options.cumulative) { - dates = (1 + maxDateSlots) * rows.length; - } - var type = this.options.cumulative ? Uint32Array: Uint8Array; + + // var type = Uint8Array; // reserve memory for all the dates - var timeIndex = new Int32Array(maxDateSlots + 1); //index-size - var timeCount = new Int32Array(maxDateSlots + 1); - var renderData = new (this.options.valueDataType || type)(dates); - var renderDataPos = new Uint32Array(dates); + // var timeIndex = new Int32Array(maxDateSlots + 1); //index-size + // var timeCount = new Int32Array(maxDateSlots + 1); + var renderData = new Array() + // var renderDataPos = new Uint32Array(dates); prof_mem.inc( 4 * maxDateSlots + // timeIndex @@ -123,73 +121,29 @@ var Profiler = require('../profiler'); // precache pixel positions for (var r = 0; r < rows.length; ++r) { var row = rows[r]; - x[r] = row.x__uint8 * this.options.resolution; + x[r] = row.x //* this.options.resolution; // fix value when it's in the tile EDGE // TODO: this should be fixed in SQL query - if (row.y__uint8 === -1) { + if (row.y === -1) { y[r] = 0; } else { - y[r] = row.y__uint8 * this.options.resolution; + y[r] = row.y //* this.options.resolution; } - var dates = row.dates__uint16; - var vals = row.vals__uint8; - if (!this.options.cumulative) { - for (var j = 0, len = dates.length; j < len; ++j) { - var rr = rowsPerSlot[dates[j]] || (rowsPerSlot[dates[j]] = []); - if(this.options.cumulative) { - vals[j] += prev_val; - } - prev_val = vals[j]; - rr.push([r, vals[j]]); - } - } else { - var valByDate = {} - for (var j = 0, len = dates.length; j < len; ++j) { - valByDate[dates[j]] = vals[j]; - } - var accum = 0; - - // extend the latest to the end - for (var j = dates[0]; j <= maxDateSlots; ++j) { - var rr = rowsPerSlot[j] || (rowsPerSlot[j] = []); - var v = valByDate[j]; - if (v) { - accum += v; - } - rr.push([r, accum]); - } - + for(var i=0; i < row.steps.length; i++){ + renderData[r*row.steps.length + i] = row.vals[i] + } /*var lastDateSlot = dates[dates.length - 1]; for (var j = lastDateSlot + 1; j <= maxDateSlots; ++j) { var rr = rowsPerSlot[j] || (rowsPerSlot[j] = []); rr.push([r, prev_val]); } */ - } } // for each timeslot search active buckets - var renderDataIndex = 0; - var timeSlotIndex = 0; - var i = 0; - for(var i = 0; i <= maxDateSlots; ++i) { - var c = 0; - var slotRows = rowsPerSlot[i] - if(slotRows) { - for (var r = 0; r < slotRows.length; ++r) { - var rr = slotRows[r]; - ++c; - renderDataPos[renderDataIndex] = rr[0] - renderData[renderDataIndex] = rr[1]; - ++renderDataIndex; - } - } - timeIndex[i] = timeSlotIndex; - timeCount[i] = c; - timeSlotIndex += c; - } + prof_process_time.end(); @@ -202,11 +156,11 @@ var Profiler = require('../profiler'); y: coord.y, z: zoom }, - timeCount: timeCount, - timeIndex: timeIndex, - renderDataPos: renderDataPos, + // timeCount: timeCount, + // timeIndex: timeIndex, + // renderDataPos: renderDataPos, renderData: renderData, - maxDate: maxDateSlots + // maxDate: maxDateSlots }; }, _generateFilterSQLForCat:function(name,categories){ @@ -225,7 +179,7 @@ var Profiler = require('../profiler'); _setFilters:function(filters){ this.filters = filters }, - _generateFiltersSQL:function(){ + _generateFiltersSQL:function(){ return Object.keys(this._filters).map(function(filterName){ var filter = this._filters[filterName] if(filter){ @@ -242,7 +196,7 @@ var Profiler = require('../profiler'); else{ return "" } - }) + }).join(" and ") }, _host: function() { var opts = this.options; @@ -358,30 +312,31 @@ var Profiler = require('../profiler'); column_conv = format("date_part('epoch', {column})", this.options); } - var sql = "" + - "WITH " + - "par AS (" + - " SELECT CDB_XYZ_Resolution({zoom})*{resolution} as res" + - ", 256/{resolution} as tile_size" + - ", CDB_XYZ_Extent({x}, {y}, {zoom}) as ext " + - ")," + - "cte AS ( "+ - " SELECT ST_SnapToGrid(i.the_geom_webmercator, p.res) g" + - ", avg(passenger_count) c1, avg(tip_amount) c2, sum( case when payment_type=1 then 1 else 0 end) c3, sum(case when payment_type=2 then 1 else 0 end ) c4, sum(case when payment_type=3 then 1 else 0 end ) c5,sum(case when payment_type=4 then 1 else 0 end ) c6, count(cartodb_id) c7 " + - " FROM ({_sql}) i, par p " + - " WHERE i.the_geom_webmercator && p.ext " + - " GROUP BY g" + - ") " + - "" + - "SELECT (st_x(g)-st_xmin(p.ext))/p.res x__uint8, " + - " (st_y(g)-st_ymin(p.ext))/p.res y__uint8," + - " Array[c1,c2,c3,c4,c5,c6,c7] vals__uint8," + - " Array[0,1,2,3,4,5,6] dates__uint16" + - // the tile_size where are needed because the overlaps query in cte subquery includes the points - // in the left and bottom borders of the tile - " FROM cte, par p where (st_y(g)-st_ymin(p.ext))/p.res < tile_size and (st_x(g)-st_xmin(p.ext))/p.res < tile_size "; - - + // var sql = "" + + // "WITH " + + // "par AS (" + + // " SELECT CDB_XYZ_Resolution({zoom})*{resolution} as res" + + // ", 256/{resolution} as tile_size" + + // ", CDB_XYZ_Extent({x}, {y}, {zoom}) as ext " + + // ")," + + // "cte AS ( "+ + // " SELECT ST_SnapToGrid(i.the_geom_webmercator, p.res) g" + + // ", avg(passenger_count) c1, avg(tip_amount) c2, sum( case when payment_type=1 then 1 else 0 end) c3, sum(case when payment_type=2 then 1 else 0 end ) c4, sum(case when payment_type=3 then 1 else 0 end ) c5,sum(case when payment_type=4 then 1 else 0 end ) c6, count(cartodb_id) c7 " + + // " FROM ({_sql}) i, par p " + + // " WHERE i.the_geom_webmercator && p.ext " + + // " GROUP BY g" + + // ") " + + // "" + + // "SELECT (st_x(g)-st_xmin(p.ext))/p.res x__uint8, " + + // " (st_y(g)-st_ymin(p.ext))/p.res y__uint8," + + // " Array[c1,c2,c3,c4,c5,c6,c7] vals__uint8," + + // " Array[0,1,2,3,4,5,6] dates__uint16" + + // // the tile_size where are needed because the overlaps query in cte subquery includes the points + // // in the left and bottom borders of the tile + // " FROM cte, par p where (st_y(g)-st_ymin(p.ext))/p.res < tile_size and (st_x(g)-st_xmin(p.ext))/p.res < tile_size "; + this.options.filters = this._generateFiltersSQL() + var sql = ""+ + "SELECT * FROM torque_tile(x:={x}, y:={y}, z:={zoom}, aggr:=ARRAY['{columns}'], table_name:='{table}', where_clause:='{filters}');" var query = format(sql, this.options, { zoom: zoom, x: coord.x, @@ -392,7 +347,9 @@ var Profiler = require('../profiler'); var self = this; + console.log("query is ", query) this.sql(query, function (data) { + if (data) { var rows = JSON.parse(data.responseText).rows; callback(self.proccessTile(rows, coord, zoom)); diff --git a/lib/torque/renderer/point.js b/lib/torque/renderer/point.js index 6153e15e..e340a5ef 100644 --- a/lib/torque/renderer/point.js +++ b/lib/torque/renderer/point.js @@ -241,6 +241,7 @@ var Filters = require('./torque_filters'); // the torque tile // _renderTile: function(tile, key, frame_offset, sprites, shader, shaderVars) { + if (!this._canvas) return; var prof = Profiler.metric('torque.renderer.point.renderTile').start(); var ctx = this._ctx; @@ -259,7 +260,7 @@ var Filters = require('./torque_filters'); var pixelIndex = 0;//tile.timeIndex[key]; for(var p = 0; p < activePixels; ++p) { if(tile.renderFlags[p]){ - var posIdx = tile.renderDataPos[pixelIndex + p]; + var posIdx = p// tile.renderDataPos[pixelIndex + p]; var c = tile.renderData[pixelIndex + p]; if (c) { var sp = sprites[c]; From aafbf83d6c0129cadead413fc5e138a31e6d257c Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 6 Nov 2015 17:06:45 +0000 Subject: [PATCH 6/6] specifying initial columns in the cartoons --- examples/bi.html | 6 +++--- lib/torque/leaflet/torque.js | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/bi.html b/examples/bi.html index 6ddb0a57..ff7bba55 100644 --- a/examples/bi.html +++ b/examples/bi.html @@ -29,8 +29,8 @@ //color scale from http://colorbrewer2.org/ var CARTOCSS = [ 'Map {', - '-torque-time-attribute: "tpep_dropoff_datetime";', - '-torque-aggregation-function: "avg(temp::float)";', + '-torque-time-attribute: "transaction_date";', + '-torque-aggregation-function: "count(1);avg(confidence_score);merchant_name";', '-torque-frame-count: 1;', '-torque-animation-duration: 15;', '-torque-resolution: 1', @@ -63,7 +63,7 @@ var torqueLayer = new L.TorqueLayer({ user : 'stuartlynn', - table : 'sampled_taxi_data', + table : 'yodlee_512', cartocss: CARTOCSS, provider: "filterable_sql_api" }); diff --git a/lib/torque/leaflet/torque.js b/lib/torque/leaflet/torque.js index ab7d98fa..e1ff9c8e 100644 --- a/lib/torque/leaflet/torque.js +++ b/lib/torque/leaflet/torque.js @@ -30,10 +30,15 @@ L.TorqueLayer = L.CanvasLayer.extend({ this.key = 0; this.prevRenderedKey = 0; this._filters = {} + if (options.cartocss) { torque.extend(options, torque.common.TorqueLayer.optionsFromCartoCSS(options.cartocss)); } + + options.columns = options.countby.split(";") + + options.resolution = options.resolution || 2; options.steps = options.steps || 100; options.visible = options.visible === undefined ? true: options.visible;