Vaex server

Why

There are various cases where the calculations and/or aggregations need to happen on a different computer than where the (aggregated) data is needed. For instance, when making a dashboard, the dashboard server might not be powerful enough for the calculations. Another example is where the client lives in a different process, such as a browser.

Starting the dataframe server

Use our server first

You can skip running your own server and first try out using https://dataframe.vaex.io

The vaex (web) server can be started from the command line like:

$ vaex server --port 8082 /data/taxi/yellow_taxi_2012.hdf5 gaia=/data/gaia/gaia-edr3-x-ps1.hdf5
INFO:MainThread:vaex.server:yellow_taxi_2012:  http://0.0.0.0:8082/dataset/yellow_taxi_2012 for REST or ws://0.0.0.0:8082/yellow_taxi_2012 for websocket
INFO:MainThread:vaex.server:gaia:  http://0.0.0.0:8082/dataset/gaia for REST or ws://0.0.0.0:8082/gaia for websocket
INFO:     Started server process [617048]
INFO:     Waiting for application startup.
INFO:     Application startup complete.
INFO:     Uvicorn running on http://0.0.0.0:8082 (Press CTRL+C to quit)

Pass files on the command line, or query help by passing the --help flag.

Python API

When the client is a Python program, the easiest API is the remote dataframe in the vaex packages itself. This does not use the REST API, but communicates over a websocket for low latency bi-directional communication.

import vaex
# the data is kept remote
df = vaex.open('vaex+wss://dataframe.vaex.io/example')
# only the result of the aggregations are send over the wire
df.x.mean()

This means you can use almost all features of a normal (local) Vaex dataframe, without having to download the data.

REST API

When the client is non-Python, or when you want to avoid the vaex dependecy, the REST API can be used.

A Vaex server is running at dataframe.vaex.io and it’s API documentation can be browsed at https://dataframe.vaex.io/docs

Some endpoints can be easily queries using curl

$ curl -i https://dataframe.vaex.io/histogram/example/x\?shape\=16
HTTP/1.1 200 OK
Server: nginx/1.18.0 (Ubuntu)
Date: Thu, 01 Apr 2021 11:23:16 GMT
Content-Type: application/json
Content-Length: 430
Connection: keep-alive
x-process-time: 0.03632664680480957
x-data-passes: 2

{"dataset_id":"example","centers":[-71.61332178115845,-58.57391309738159,-45.534504413604736,-32.49509572982788,-19.455687046051025,-6.41627836227417,6.6231303215026855,19.66253900527954,32.7019476890564,45.74135637283325,58.78076505661011,71.82017374038696,84.85958242416382,97.89899110794067,110.93839979171753,123.97780847549438],"values":[3.0,0.0,3.0,917.0,13706.0,154273.0,147171.0,12963.0,960.0,1.0,1.0,0.0,0.0,0.0,1.0,1.0]}

While the POST method might be more convenient from a Javascript client or using a HTTP Library.

Python using requests

Requests is an easy to use HTTP library.

import requests
data = {
    'dataset_id': 'gaia-dr2',
    'expression_x': 'l',
    'expression_y': 'b',
    'filter': None,
    'virtual_columns': [],
    'min_x': 0,
    'max_x': 360,
    'min_y': -90,
    'max_y': 90,
    'shape_x': 512,
    'shape_y': 256,
}
response = requests.post('https://dataframe.vaex.io/heatmap', json=data)
response.json()
assert response.status_code == 200, 'oops, something went wrong'
{'dataset_id': 'gaia-dr2',
 'centers_x': [22.5, 67.5, 112.5, 157.5, 202.5, 247.5, 292.5, 337.5],
 'centers_y': [-67.5, -22.5, 22.5, 67.5],
 'values': [[3508786.0,
   2711710.0,
   2287021.0,
   2042114.0,
   2009057.0,
   2448207.0,
   3716951.0,
   3644323.0],
  [250883466.0,
   100064757.0,
   49538929.0,
   28273970.0,
   30521201.0,
   53214391.0,
   159460735.0,
   251170124.0],
  [166984543.0,
   110774989.0,
   43475771.0,
   31343345.0,
   31584354.0,
   44061582.0,
   108436851.0,
   189699927.0],
  [3388522.0,
   2848641.0,
   2221241.0,
   1997993.0,
   1941090.0,
   2215271.0,
   3061986.0,
   3387287.0]]}
var inputData = {
    dataset_id: 'gaia-dr2',
    expression_x: 'l',
    expression_y: 'b',
    filter: null,
    virtual_columns: [],
    min_x: 0,
    max_x: 360,
    min_y: -90,
    max_y: 90,
    shape: [512, 256],
};
var result = await fetch("https://dataframe.vaex.io/heatmap", {method: 'POST', body: JSON.stringify(inputData)})
var data = await result.json();
console.log(data);


{dataset_id: "gaia-dr2", centers_x: Array(512), centers_y: Array(256), values: Array(256)}
centers_x: (512) [0.3515625, 1.0546875, 1.7578125, 2.4609375, 3.1640625, 3.8671875, 4.5703125, 5.2734375, 5.9765625, 6.6796875, 7.3828125, 8.0859375, 8.7890625, 9.4921875, 10.1953125, 10.8984375, 11.6015625, 12.3046875, 13.0078125, 13.7109375, 14.4140625, 15.1171875, 15.8203125, 16.5234375, 17.2265625, 17.9296875, 18.6328125, 19.3359375, 20.0390625, 20.7421875, 21.4453125, 22.1484375, 22.8515625, 23.5546875, 24.2578125, 24.9609375, 25.6640625, 26.3671875, 27.0703125, 27.7734375, 28.4765625, 29.1796875, 29.8828125, 30.5859375, 31.2890625, 31.9921875, 32.6953125, 33.3984375, 34.1015625, 34.8046875, 35.5078125, 36.2109375, 36.9140625, 37.6171875, 38.3203125, 39.0234375, 39.7265625, 40.4296875, 41.1328125, 41.8359375, 42.5390625, 43.2421875, 43.9453125, 44.6484375, 45.3515625, 46.0546875, 46.7578125, 47.4609375, 48.1640625, 48.8671875, 49.5703125, 50.2734375, 50.9765625, 51.6796875, 52.3828125, 53.0859375, 53.7890625, 54.4921875, 55.1953125, 55.8984375, 56.6015625, 57.3046875, 58.0078125, 58.7109375, 59.4140625, 60.1171875, 60.8203125, 61.5234375, 62.2265625, 62.9296875, 63.6328125, 64.3359375, 65.0390625, 65.7421875, 66.4453125, 67.1484375, 67.8515625, 68.5546875, 69.2578125, 69.9609375, …]
centers_y: (256) [-89.6484375, -88.9453125, -88.2421875, -87.5390625, -86.8359375, -86.1328125, -85.4296875, -84.7265625, -84.0234375, -83.3203125, -82.6171875, -81.9140625, -81.2109375, -80.5078125, -79.8046875, -79.1015625, -78.3984375, -77.6953125, -76.9921875, -76.2890625, -75.5859375, -74.8828125, -74.1796875, -73.4765625, -72.7734375, -72.0703125, -71.3671875, -70.6640625, -69.9609375, -69.2578125, -68.5546875, -67.8515625, -67.1484375, -66.4453125, -65.7421875, -65.0390625, -64.3359375, -63.6328125, -62.9296875, -62.2265625, -61.5234375, -60.8203125, -60.1171875, -59.4140625, -58.7109375, -58.0078125, -57.3046875, -56.6015625, -55.8984375, -55.1953125, -54.4921875, -53.7890625, -53.0859375, -52.3828125, -51.6796875, -50.9765625, -50.2734375, -49.5703125, -48.8671875, -48.1640625, -47.4609375, -46.7578125, -46.0546875, -45.3515625, -44.6484375, -43.9453125, -43.2421875, -42.5390625, -41.8359375, -41.1328125, -40.4296875, -39.7265625, -39.0234375, -38.3203125, -37.6171875, -36.9140625, -36.2109375, -35.5078125, -34.8046875, -34.1015625, -33.3984375, -32.6953125, -31.9921875, -31.2890625, -30.5859375, -29.8828125, -29.1796875, -28.4765625, -27.7734375, -27.0703125, -26.3671875, -25.6640625, -24.9609375, -24.2578125, -23.5546875, -22.8515625, -22.1484375, -21.4453125, -20.7421875, -20.0390625, …]
dataset_id: "gaia-dr2"
values: (256) [ …]
__proto__: Object

Example using plotly.js

Combining the previous with the plotly.js library we can make an interactive plot:

Sky map

First, make sure we have a div

<div id="plotlyHeatmap"></div>

Then load the data, and plot it using plotly.js:

var skyMapInput = {
    dataset_id: 'gaia-dr2',
    expression_x: 'l',
    expression_y: 'b',
    virtual_columns: {
        distance: "1/parallax"
    },
    filter: this.filter,
    min_x: 0,
    max_x: 360,
    min_y: -90,
    max_y: 90,
    shape: [512, 256],
};

async function loadData(heatmapInput) {
    const result = await fetch("https://dataframe-dev.vaex.io/heatmap", {method: 'POST', body: JSON.stringify(heatmapInput)})
    const data = await result.json();
    return data;
}

function plotData(elementId, data, log, xaxis, yaxis) {
    const trace_data = {
        x: data.centers_x,
        y: data.centers_y,
        z: log ? data.values.map((ar1d) => ar1d.map(Math.log1p)) : data.values,
        type: 'heatmap',
        colorscale: 'plasma',
        transpose: true,
    };
    var layout = {
        xaxis: {
            title: {
                text: data.expression_x,
            },
            ...xaxis
        },
        yaxis: {
            title: {
                text: data.expression_y,
            },
            ...yaxis
        }
    };
    Plotly.react(elementId, [trace_data], layout);
}

async function plot(elementId, heatmapInput, xaxis, yaxis) {
    const heatmapOutput = await loadData(heatmapInput);
    await plotData(elementId, heatmapOutput, true, xaxis, yaxis);
}

plot('plotlyHeatmap', skyMapInput);

Adding an event handler, will refine the data when we zoom in:

function addZoomHandler(elementId, heatmapInput) {
    document.getElementById(elementId).on('plotly_relayout', async (e) => {
        // mutate input data
        heatmapInput.min_x = e["xaxis.range[0]"]
        heatmapInput.max_x = e["xaxis.range[1]"]
        heatmapInput.min_y = e["yaxis.range[0]"]
        heatmapInput.max_y = e["yaxis.range[1]"]
        // and plot again
        plot(elementId, heatmapInput);
    })
}

CMD

We can now easily add a second heatmap

<div id="plotlyHeatmapCMD"></div>

And plot a different heatmap (a color-magnitude diagram) on this div.

var cmdInput = {
    dataset_id: 'gaia-dr2',
    expression_x: 'phot_bp_mean_mag-phot_rp_mean_mag',
    expression_y: 'M_g',
    virtual_columns: {
        distance: "1/parallax",
        M_g: "phot_g_mean_mag-(5*log10(distance)+10)"
    },
    filter: '((pmra**2+pmdec**2)<100)&(parallax_over_error>10)&(abs(b)>20)',
    min_x: -1,
    max_x: 5,
    min_y: 15,
    max_y: -5,
    shape_x: 256,
    shape_y: 256,
};

async () => {
    await plot('plotlyHeatmapCMD', cmdInput);
    addZoomHandler('plotlyHeatmapCMD', cmdInput);
}