How to paginate in Node.js using Azure Table Storage
Imagine you have millions of entities in an Azure table, and you want to page through them displaying 20 entities at a time. Fetching all entities and then dividing them into groups of 20 is hardly the most efficient way to do this. In addition, at the time this document is being written, Azure limits the number of returned entities to 1000 in any given request. Among other things, this keeps developers from accidentally querying for millions of entities.
Azure Table Storage supports continuation tokens to support paging through a large number of entities. You fetch a group entities, and the result contains a continuation token if there are more entities remaining to be fetched. The continuation token is like a bookmark which indicates where the query left off. For example, if you fetched entities 1-20, a continuation token is included to tell you to begin your next query at entity number 21. This is how you can efficiently page through a large number of entities.
The code at the bottom of this post illustrates how to do pagination in Node with Azure tables. Save it in a file called table-storage-pagination-sample.js. It creates a sample database containing totalEntities number of entities. It then queries the results and displays pageSize entities per page.
createTableIfNotExists() simply creates a table and populates it with sample data using a batch insertion. These can include at most 100 entities at a time, so the code loops through this operation until the number of entities in totalEntities has been created. Make totalEntities a multiple of 100 so that totalEntities / 100 is an integer. This indicates how many batches of 100 insertions should be performed.
In http.createServer() we ignore requests for /favicon.ico for the sake of simplicity.
Skip over the if block which handles requests to /nextPage for now. We'll come back to it.
This line defines our initial query:
var query = azure.TableQuery.select().from(tableName).top(pageSize);
You've probably seen select() and from() before. top() limits the query to pageSize number of entities.
In the parameter list for entitiesQueriedCallback, note that we now include pageContinuation. This object contains the continuation token. If you're curious, set a breakpoint or use console.log() to inspect pageContinuation. You'll see that it contains these properties: tableService, tableQuery, nextPartitionKey, and nextRowKey.
entitiesQueriedCallback loops through the results and uses counter to keep track of how many total results have so far been returned. counter is initialized at the top of http.createServer() because it's also used by the if block which handles requests for /nextPage.
If there are more entities yet to be retrieved, pageContinuation.hasNextPage() will return true. If that's the case, then we emit a link to /nextPage which includes nextPartitionKey and nextRowKey as query strings.
In the if block which handles requests for /nextPage, we use the querystring module to extract nextPartitionKey and nextRowKey from the requested URL. We then create nextPageQuery which contains these keys and pass it to queryEntities().
When there are no more entities to be retrieved, pageContinuation.hasNextPage() returns false and we no longer display a link for the next page.
Here’s the code:
var http = require('http');
var url = require('url');
var querystring = require('querystring');
var azure = require('azure');
var uuid = require('node-uuid');
var port = process.env.PORT || 1337; // for Azure or when running locally.
var account = 'your account goes here';
var accountKey = 'your account key goes here';
var tableService = azure.createTableService(account, accountKey);
// Uncomment this to enable logging
//tableService.logger = new azure.Logger(azure.Logger.LogLevels.DEBUG);
var totalEntities = 100; // Num of entities to create. Make it multiple of 100.
var pageSize = 20;
var tableName = 'largetable'; // Name your table here.
// Create and populate the table. Note that for batch operations,
// the PartitionKey must be the same for all entities.
tableService.createTableIfNotExists(tableName, function (err, created) {
if (created) {
var numberOfBatches = parseInt(totalEntities / 100);
for (var i = 0; i < numberOfBatches; i++) {
tableService.beginBatch();
var now = new Date().toString();
// Batches can include at most 100 entities.
for (var j = 0; j < 100; j++) {
tableService.insertEntity(tableName, {
PartitionKey: 'White', RowKey: (i * 100 + j + 1).toString(),
Winery: 'Azure Vineyards', Variety: 'Chardonnay',
Vintage: '2003', Notes: uuid(), TastedOn: now
});
}
tableService.commitBatch(function () {
console.log('Initialized table "' + tableName +
'" with 100 sample entities.');
});
}
}
});
http.createServer(function (req, res) {
var counter = 0;
if (req.url === '/favicon.ico') return;
if (url.parse(req.url).pathname === '/nextPage') {
var parsedQuerystring = querystring.parse(url.parse(req.url).query);
var nextPartitionKey = parsedQuerystring.nextPartitionKey;
var nextRowKey = parsedQuerystring.nextRowKey;
var nextPageQuery = azure.TableQuery.select().from(tableName)
.top(pageSize).whereNextKeys(nextPartitionKey, nextRowKey);
tableService.queryEntities(nextPageQuery, entitiesQueriedCallback);
return;
}
var query = azure.TableQuery.select().from(tableName).top(pageSize);
tableService.queryEntities(query, entitiesQueriedCallback);
function entitiesQueriedCallback(error, serverEntities, pageContinuation) {
res.writeHead(200, { 'Content-Type': 'text/html' });
if (error === null) {
for (var index in serverEntities) {
res.write('RowKey: ' + serverEntities[index].RowKey +
', Winery: ' + serverEntities[index].Winery + ', Notes: ' +
serverEntities[index].Notes + '<br />');
counter++;
}
res.write('<br />Returned ' + counter + ' entities. <br />');
if (pageContinuation.hasNextPage()) {
res.write(
'<a href="https://blogs.msdn.com/nextPage?nextPartitionKey='
+ pageContinuation.nextPartitionKey +
'&nextRowKey=' +
pageContinuation.nextRowKey + '">Next page</a>');
}
res.end();
} else {
res.end('Could not query entities: ' + error.code);
console.log('Could not query entities: ' + error.code);
}
}
}).listen(port);
Post made with MarkdownPad.