HTTP API for Latest Covid-19 Data
@pravin
code:
data:privatelast updated:1 year ago
today
Build integrations remarkably fast!
You're viewing a public workflow template.
Sign up to customize, add steps, modify code and more.
Join 200,000+ developers using the Pipedream platform
steps.
trigger
HTTP API
Deploy to generate unique URL
This workflow runs on Pipedream's servers and is triggered by HTTP / Webhook requests.
steps.
readme
auth
to use OAuth tokens and API keys in code via theauths object
code
Write any Node.jscodeand use anynpm package. You can alsoexport datafor use in later steps via return or this.key = 'value', pass input data to your code viaparams, and maintain state across invocations with$checkpoint.
async (event, steps) => {
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
}
32
/*
This workflow provides an HTTP API to retrieve data in 
JSON format about the number of cases, recoveries and 
deaths by region related to the 2019-nCoV Wuhan coronavirus.
The source data is aggregated and published to Github by 
the The Center for Systems Science and Engineering (CSSE) 
at John Hopkins University.
 
To use this API, make an HTTP request to:

https://coronavirus.m.pipedream.net

The data includes:

1. Summary stats (counts of cases, recoveries and deaths)
   - Global
   - China
   - Non-China

2. Raw data (counts by region as published in the Google Sheet)

Data is cached using $checkpoint to improve performance. The 
cache is updated if it's more than 5 minutes old.

Data source: https://github.com/CSSEGISandData/COVID-19/tree/master/csse_covid_19_data/csse_covid_19_daily_reports

NOTE: The previous version of this API used data published to
Google Sheets. That data is no longer maintained. The source
code for that API is at https://pipedream.com/@pravin/backup-http-api-for-latest-wuhan-coronavirus-data-2019-ncov-p_QPCd3j/edit
*/
steps.
filter_favicon_requests
auth
to use OAuth tokens and API keys in code via theauths object
code
Write any Node.jscodeand use anynpm package. You can alsoexport datafor use in later steps via return or this.key = 'value', pass input data to your code viaparams, and maintain state across invocations with$checkpoint.
async (event, steps) => {
1
2
3
4
5
}
6
// filter out requests for favicon assets
if (steps.trigger.event.url.indexOf('favicon.ico') !== -1) {
  $end('Terminating workfow for favicon request')
}
steps.
get_data
auth
to use OAuth tokens and API keys in code via theauths object
params
Owner

The ID of the spreadsheet to insert rows into. The spreadsheetID can be found in the URL when viewing your Google sheet. E.g., https://docs.google.com/spreadsheets/d/[spreadsheetId]/edit#gid=0

CSSEGISandData
string ·params.owner
Repo

The A1 notation of the values to retrieve. E.g., A1:E5 or Sheet1!A1:E5

COVID-19
string ·params.repo
Path
csse_covid_19_data/csse_covid_19_daily_reports
string ·params.path
code
Write any Node.jscodeand use anynpm package. You can alsoexport datafor use in later steps via return or this.key = 'value', pass input data to your code viaparams, and maintain state across invocations with$checkpoint.
async (event, steps, params) => {
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
}
102
if (typeof this.$checkpoint === 'undefined') {
  // initialize this.$checkpoint
  this.$checkpoint = {}
  this.$checkpoint.data = {}
  this.$checkpoint.ts = 0
  this.$checkpoint.lastModified = ''
  this.$checkpoint.range = ''
  this.$checkpoint.spreadsheetId = ''
  this.$checkpoint.lastPublished = 0
}

const { Octokit }  = require("@octokit/rest");
const _ = require('lodash')
const moment = require('moment')
const axios = require('axios')
const parse = require('csv-parse/lib/sync')
const stripBom = require('strip-bom');

this.dataExpiry = 5 * 60 * 1000

// If the cache is expired or if there is a request to force an
// update, then we should update the cache
if (((new Date().getTime() - this.$checkpoint.ts) > (this.dataExpiry)) || 
    (event.query.action === 'refresh' && event.query.token === 
    process.env.CSSE_NCOV_REFRESH_TOKEN)) {
  this.updateCache = true
} else {
  this.updateCache = false
}

// Update the cache or return the data from this.$checkpoint
// based on the value of this.updateCache
if (this.updateCache === true) {

  let maxDate = 0

  // initialize octokit
  const octokit = new Octokit({
    userAgent: steps.trigger.context.workflow_id,
    baseUrl: 'https://api.github.com',
  })

  // get the files in the target repo folder
  this.rawData = (await octokit.repos.getContents({
    owner: params.owner,
    repo: params.repo,
    path: params.path,
  })).data
  
  // get the file with the most recent data
  for ( let i = 0; i < this.rawData.length; i++ ) {
    let fileData = this.rawData[i].name.split(".")
    if (fileData[1] === 'csv') {
      if (moment(fileData[0], "MM-DD-YYYY").format('x') > moment(maxDate, "MM-DD-YYYY").format('x') || maxDate === 0) {
        maxDate = fileData[0]
      }
    }
  }

  // export the date associated with the most recent data
  this.maxDate = maxDate

  // get the most recent data
  const dataSourceUrl = `https://raw.githubusercontent.com/${params.owner}/${params.repo}/master/${params.path}/${maxDate}.csv`
  const data = (await axios.get(dataSourceUrl)).data

  // strip the byte order mark
  const cleansedData = stripBom(data)

  // parse the CSV data to an array
  const parsedData = parse(cleansedData, {
    columns: true,
    skip_empty_lines: true
  })

  this.mostRecentData = 0
  parsedData.forEach( update => {
    if ( moment(update['Last Update']).format('x') > moment(this.mostRecentData).format('x') || this.mostRecentData === 0) {
      this.mostRecentData = update['Last Update']
    }
  })

  // get commits to identify when the file was last updated
  this.commits = await octokit.repos.listCommits({
    owner: params.owner,
    repo: params.repo,
    path: `${params.path}/${maxDate}.csv`,
  });

  // save data to $checkpoint
  this.$checkpoint.dataSource = dataSourceUrl
  this.$checkpoint.data = parsedData
  this.$checkpoint.ts = new Date().getTime()
  this.$checkpoint.lastModified = this.commits.data[0].commit.committer.date
  this.$checkpoint.lastPublished = this.maxDate
} else {
  console.log('Return cached data')
}

return this.$checkpoint
steps.
transform_data
auth
to use OAuth tokens and API keys in code via theauths object
code
Write any Node.jscodeand use anynpm package. You can alsoexport datafor use in later steps via return or this.key = 'value', pass input data to your code viaparams, and maintain state across invocations with$checkpoint.
async (event, steps) => {
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
}
18
// transform the data into an array of objects with key/value pairs
const transformedData = [], originalData = steps.get_data.$return_value.data.values
let rowCount = 0

originalData.forEach(function(row){
  if (rowCount > 0) {
    let obj = {}
    for (let i=0; i<row.length; i++) {
      obj[originalData[0][i]] = row[i] 
    }
    transformedData.push(obj)
  }
  rowCount++
})

return transformedData
steps.
summarize_data
auth
to use OAuth tokens and API keys in code via theauths object
code
Write any Node.jscodeand use anynpm package. You can alsoexport datafor use in later steps via return or this.key = 'value', pass input data to your code viaparams, and maintain state across invocations with$checkpoint.
async (event, steps) => {
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
}
40
// if the cache was updated in steps.get_data, then
// recalculate the summary stats and checkpoint them.
// Otherwise, return the summary cached in this.$checkpoint
if (steps.get_data.updateCache === true) {
  console.log('updating cached stats')

  // initialize the stats object
  const stats = {
    global: { confirmed: 0, recovered: 0, deaths: 0 },
    china: { confirmed: 0, recovered: 0, deaths: 0 },
    nonChina: { confirmed: 0, recovered: 0, deaths: 0 },
  }

  function incrementTotals(statsObj, regionObj) {
    statsObj.confirmed += parseInt(regionObj.Confirmed)
    statsObj.recovered += parseInt(regionObj.Recovered)
    statsObj.deaths += parseInt(regionObj.Deaths)
    return statsObj
  }

  steps.get_data.$return_value.data.forEach(function(region){
    // increment global totals
    stats.global = incrementTotals(stats.global, region)
    if (region['Country/Region'] === 'China') {
      // calculate totals for china
      stats.china = incrementTotals(stats.china, region)
    } else {
      // calculate totals for non-china regions 
      stats.nonChina = incrementTotals(stats.nonChina, region)
    }
  })

  this.$checkpoint = stats
} else {
  console.log('using cached stats')
}

return this.$checkpoint
steps.
nodejs
auth
to use OAuth tokens and API keys in code via theauths object
code
Write any Node.jscodeand use anynpm package. You can alsoexport datafor use in later steps via return or this.key = 'value', pass input data to your code viaparams, and maintain state across invocations with$checkpoint.
async (event, steps) => {
1
2
3
4
5
6
7
8
}
9
try {
  const v8 = require('v8');
  this.heapSpaceStats = v8.getHeapSpaceStatistics()
  this.heapStats = v8.getHeapStatistics()
} catch (err) {
  this.err = err
}
steps.
respond_to_client
auth
to use OAuth tokens and API keys in code via theauths object
code
Write any Node.jscodeand use anynpm package. You can alsoexport datafor use in later steps via return or this.key = 'value', pass input data to your code viaparams, and maintain state across invocations with$checkpoint.
async (event, steps) => {
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
}
36
// construct the response body and respond to the client
// request using $respond()

const moment = require('moment')

const body = {}
const lastUpdatedTimestamp = steps.get_data.$return_value.ts
const expiresTimestamp = steps.get_data.dataExpiry 
                         + steps.get_data.$return_value.ts

body.summaryStats = steps.summarize_data.$return_value
body.cache = {
  lastUpdated: moment(lastUpdatedTimestamp).fromNow(),
  expires: moment(expiresTimestamp).fromNow(),
  lastUpdatedTimestamp,
  expiresTimestamp
}
body.dataSource = {
  url: steps.get_data.$return_value.dataSource,
  lastGithubCommit: steps.get_data.$return_value.lastModified,
  mostRecentData: steps.get_data.mostRecentData,
  publishedBy: `John Hopkins University Center for Systems Science and Engineering`,
  ref: `https://github.com/CSSEGISandData/COVID-19`
}
body.apiSourceCode = `https://pipedream.com/@/${steps.trigger.context.workflow_id}`
body.rawData = steps.get_data.$return_value.data

$respond({
  status: 200,
  headers: {
    'content-type': 'application/json'
  },
  body: JSON.stringify(body)
})