auths
objectreturn
or this.key = 'value'
, pass input data to your code viaparams
, and maintain state across executions with$checkpoint.async
(event, steps) => {
}
/*
This workflow provides an HTTP API to retrieve data in
JSON format about the number of cases, recoveries and
deaths by region related to the 2019-nCoV Wuhan coronavirus.
The source data is aggregated and published to Github by
the The Center for Systems Science and Engineering (CSSE)
at John Hopkins University.
To use this API, make an HTTP request to:
https://coronavirus.m.pipedream.net
The data includes:
1. Summary stats (counts of cases, recoveries and deaths)
- Global
- China
- Non-China
2. Raw data (counts by region as published in the Google Sheet)
Data is cached using $checkpoint to improve performance. The
cache is updated if it's more than 5 minutes old.
Data source: https://github.com/CSSEGISandData/COVID-19/tree/master/csse_covid_19_data/csse_covid_19_daily_reports
NOTE: The previous version of this API used data published to
Google Sheets. That data is no longer maintained. The source
code for that API is at https://pipedream.com/@pravin/backup-http-api-for-latest-wuhan-coronavirus-data-2019-ncov-p_QPCd3j/edit
*/
auths
objectreturn
or this.key = 'value'
, pass input data to your code viaparams
, and maintain state across executions with$checkpoint.async
(event, steps) => {
}
// filter out requests for favicon assets
if (steps.trigger.event.url.indexOf('favicon.ico') !== -1) {
$end('Terminating workfow for favicon request')
}
auths
objectThe ID of the spreadsheet to insert rows into. The spreadsheetID can be found in the URL when viewing your Google sheet. E.g., https://docs.google.com/spreadsheets/d/[spreadsheetId]/edit#gid=0
The A1 notation of the values to retrieve. E.g., A1:E5 or Sheet1!A1:E5
return
or this.key = 'value'
, pass input data to your code viaparams
, and maintain state across executions with$checkpoint.async
(event, steps, params) => {
}
if (typeof this.$checkpoint === 'undefined') {
// initialize this.$checkpoint
this.$checkpoint = {}
this.$checkpoint.data = {}
this.$checkpoint.ts = 0
this.$checkpoint.lastModified = ''
this.$checkpoint.range = ''
this.$checkpoint.spreadsheetId = ''
this.$checkpoint.lastPublished = 0
}
const { Octokit } = require("@octokit/rest");
const _ = require('lodash')
const moment = require('moment')
const axios = require('axios')
const parse = require('csv-parse/lib/sync')
const stripBom = require('strip-bom');
this.dataExpiry = 5 * 60 * 1000
// If the cache is expired or if there is a request to force an
// update, then we should update the cache
if (((new Date().getTime() - this.$checkpoint.ts) > (this.dataExpiry)) ||
(event.query.action === 'refresh' && event.query.token ===
process.env.CSSE_NCOV_REFRESH_TOKEN)) {
this.updateCache = true
} else {
this.updateCache = false
}
// Update the cache or return the data from this.$checkpoint
// based on the value of this.updateCache
if (this.updateCache === true) {
let maxDate = 0
// initialize octokit
const octokit = new Octokit({
userAgent: steps.trigger.context.workflow_id,
baseUrl: 'https://api.github.com',
})
// get the files in the target repo folder
this.rawData = (await octokit.repos.getContents({
owner: params.owner,
repo: params.repo,
path: params.path,
})).data
// get the file with the most recent data
for ( let i = 0; i < this.rawData.length; i++ ) {
let fileData = this.rawData[i].name.split(".")
if (fileData[1] === 'csv') {
if (moment(fileData[0], "MM-DD-YYYY").format('x') > moment(maxDate, "MM-DD-YYYY").format('x') || maxDate === 0) {
maxDate = fileData[0]
}
}
}
// export the date associated with the most recent data
this.maxDate = maxDate
// get the most recent data
const dataSourceUrl = `https://raw.githubusercontent.com/${params.owner}/${params.repo}/master/${params.path}/${maxDate}.csv`
const data = (await axios.get(dataSourceUrl)).data
// strip the byte order mark
const cleansedData = stripBom(data)
// parse the CSV data to an array
const parsedData = parse(cleansedData, {
columns: true,
skip_empty_lines: true
})
this.mostRecentData = 0
parsedData.forEach( update => {
if ( moment(update['Last Update']).format('x') > moment(this.mostRecentData).format('x') || this.mostRecentData === 0) {
this.mostRecentData = update['Last Update']
}
})
// get commits to identify when the file was last updated
this.commits = await octokit.repos.listCommits({
owner: params.owner,
repo: params.repo,
path: `${params.path}/${maxDate}.csv`,
});
// save data to $checkpoint
this.$checkpoint.dataSource = dataSourceUrl
this.$checkpoint.data = parsedData
this.$checkpoint.ts = new Date().getTime()
this.$checkpoint.lastModified = this.commits.data[0].commit.committer.date
this.$checkpoint.lastPublished = this.maxDate
} else {
console.log('Return cached data')
}
return this.$checkpoint
auths
objectreturn
or this.key = 'value'
, pass input data to your code viaparams
, and maintain state across executions with$checkpoint.async
(event, steps) => {
}
// transform the data into an array of objects with key/value pairs
const transformedData = [], originalData = steps.get_data.$return_value.data.values
let rowCount = 0
originalData.forEach(function(row){
if (rowCount > 0) {
let obj = {}
for (let i=0; i<row.length; i++) {
obj[originalData[0][i]] = row[i]
}
transformedData.push(obj)
}
rowCount++
})
return transformedData
auths
objectreturn
or this.key = 'value'
, pass input data to your code viaparams
, and maintain state across executions with$checkpoint.async
(event, steps) => {
}
// if the cache was updated in steps.get_data, then
// recalculate the summary stats and checkpoint them.
// Otherwise, return the summary cached in this.$checkpoint
if (steps.get_data.updateCache === true) {
console.log('updating cached stats')
// initialize the stats object
const stats = {
global: { confirmed: 0, recovered: 0, deaths: 0 },
china: { confirmed: 0, recovered: 0, deaths: 0 },
nonChina: { confirmed: 0, recovered: 0, deaths: 0 },
}
function incrementTotals(statsObj, regionObj) {
statsObj.confirmed += parseInt(regionObj.Confirmed)
statsObj.recovered += parseInt(regionObj.Recovered)
statsObj.deaths += parseInt(regionObj.Deaths)
return statsObj
}
steps.get_data.$return_value.data.forEach(function(region){
// increment global totals
stats.global = incrementTotals(stats.global, region)
if (region['Country/Region'] === 'China') {
// calculate totals for china
stats.china = incrementTotals(stats.china, region)
} else {
// calculate totals for non-china regions
stats.nonChina = incrementTotals(stats.nonChina, region)
}
})
this.$checkpoint = stats
} else {
console.log('using cached stats')
}
return this.$checkpoint
auths
objectreturn
or this.key = 'value'
, pass input data to your code viaparams
, and maintain state across executions with$checkpoint.async
(event, steps) => {
}
try {
const v8 = require('v8');
this.heapSpaceStats = v8.getHeapSpaceStatistics()
this.heapStats = v8.getHeapStatistics()
} catch (err) {
this.err = err
}
auths
objectreturn
or this.key = 'value'
, pass input data to your code viaparams
, and maintain state across executions with$checkpoint.async
(event, steps) => {
}
// construct the response body and respond to the client
// request using $respond()
const moment = require('moment')
const body = {}
const lastUpdatedTimestamp = steps.get_data.$return_value.ts
const expiresTimestamp = steps.get_data.dataExpiry
+ steps.get_data.$return_value.ts
body.summaryStats = steps.summarize_data.$return_value
body.cache = {
lastUpdated: moment(lastUpdatedTimestamp).fromNow(),
expires: moment(expiresTimestamp).fromNow(),
lastUpdatedTimestamp,
expiresTimestamp
}
body.dataSource = {
url: steps.get_data.$return_value.dataSource,
lastGithubCommit: steps.get_data.$return_value.lastModified,
mostRecentData: steps.get_data.mostRecentData,
publishedBy: `John Hopkins University Center for Systems Science and Engineering`,
ref: `https://github.com/CSSEGISandData/COVID-19`
}
body.apiSourceCode = `https://pipedream.com/@/${steps.trigger.context.workflow_id}`
body.rawData = steps.get_data.$return_value.data
$respond({
status: 200,
headers: {
'content-type': 'application/json'
},
body: JSON.stringify(body)
})