COVID19 India Vaccination CSV Data - Simple Serverless Service
@thejeshgn
code:
data:privatelast updated:1 year ago
today
Build integrations remarkably fast!
You're viewing a public workflow template.
Sign up to customize, add steps, modify code and more.
Join 250,000+ developers using the Pipedream platform
steps.
trigger
HTTP API
Deploy to generate unique URL
This workflow runs on Pipedream's servers and is triggered by HTTP / Webhook requests.
steps.
return_response
auth
to use OAuth tokens and API keys in code via theauths object
code
Write any Node.jscodeand use anynpm package. You can alsoexport datafor use in later steps via return or this.key = 'value', pass input data to your code viaparams, and maintain state across invocations with$checkpoint.
async (event, steps) => {
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
}
157
//########################################################################################
// Thejesh GN https://thejeshgn.com
//
// This converts the JSON into CSV so it can be used by DataWrapper
// This is a hack where I combined piecies of code from various projects
// I have added link to those below if you want to check them out
//
// I love JSON to CSV converter by Eric Mill, https://konklone.io/json/ 
// Source code of it is at https://github.com/konklone/json
// It does an awesome job of converting JSON to CSV in the front end
//
// I wanted to do the same in backend and the pipedream seemed like a good place
// To host that serverless code. at some point I will write a proper serverless code
// Where if I send the url of JSON it will return CSV. Until then this hack is good 
// enough for me.
//
// The final datawrapper graph is at https://datawrapper.dwcdn.net/ThC0E/2/
// It caches the data, so its not a major hit on pipedream.
//########################################################################################
const axios = require('axios')

//########################################################################################
// Pass in the objects to merge as arguments.
// For a deep extend, set the first argument to `true`.
// https://gomakethings.com/vanilla-javascript-version-of-jquery-extend/
var extend = function () {

    // Variables
    var extended = {};
    var deep = false;
    var i = 0;
    var length = arguments.length;

    // Check if a deep merge
    if ( Object.prototype.toString.call( arguments[0] ) === '[object Boolean]' ) {
        deep = arguments[0];
        i++;
    }

    // Merge the object into the extended object
    var merge = function (obj) {
        for ( var prop in obj ) {
            if ( Object.prototype.hasOwnProperty.call( obj, prop ) ) {
                // If deep merge and property is an object, merge properties
                if ( deep && Object.prototype.toString.call(obj[prop]) === '[object Object]' ) {
                    extended[prop] = extend( true, extended[prop], obj[prop] );
                } else {
                    extended[prop] = obj[prop];
                }
            }
        }
    };

    // Loop through each object and conduct a merge
    for ( ; i < length; i++ ) {
        var obj = arguments[i];
        merge(obj);
    }

    return extended;

};

//########################################################################################
//https://konklone.io/json/
//https://github.com/konklone/json/blob/gh-pages/assets/site.js
function parse_object(obj, path) {
    if (path == undefined)
        path = "";

    var type = typeof(obj);
    var scalar = (type == "number" || type == "string" || type == "boolean" || type == "null");

    if (type == "array" || type == "object") {
        var d = {};
        for (var i in obj) {

            var newD = parse_object(obj[i], path + i + "/");
            d = extend(d, newD);
        }

        return d;
    }

    else if (scalar) {
        var d = {};
        var endPath = path.substr(0, path.length-1);
        d[endPath] = obj;
        return d;
    }

    // ?
    else return {};
}

// otherwise, just find the first one
function arrayFrom(json) {
    var queue = [], next = json;
    while (next !== undefined) {
        if (typeof(next) == "array") {

            // but don't if it's just empty, or an array of scalars
            if (next.length > 0) {

              var type = typeof(next[0]);
              var scalar = (type == "number" || type == "string" || type == "boolean" || type == "null");

              if (!scalar)
                return next;
            }
        } if (typeof(next) == "object") {
          for (var key in next)
             queue.push(next[key]);
        }
        next = queue.shift();
    }
    // none found, consider the whole object a row
    return [json];
}

function removeTrailingComma(input) {
  if (input.slice(-1) == ",")
    return input.slice(0,-1);
  else
    return input;
}

function flatten(inArray) {
    var outArray = [];
    for (var row in inArray)
        outArray[outArray.length] = parse_object(inArray[row]);
    return outArray
}
//########################################################################################
//https://www.newline.co/@anthonygore/how-to-convert-json-to-csv-in-nodejs--23c6b226
function convert_to_csv(data){
    csv = data.map(row => Object.values(row));
    csv.unshift(Object.keys(data[0]));
    csv = csv.join('\n');
    console.log(csv)
    return csv
}
//########################################################################################
const dataSourceUrl = "https://raw.githubusercontent.com/datameet/covid19/master/data/mohfw_vaccination_status.json"
const data_json =  (await axios.get(dataSourceUrl)).data
const data = flatten(data_json.rows)
console.log(data)


$respond({
  immediate: true,
  status: 200,
  headers: {"Content-type":"text/plain"},
  body: convert_to_csv(data)
})
//########################################################################################
// Thejesh GN https://thejeshgn.com
//
// This converts the JSON into CSV so it can be used by DataWrapper
// This is a hack where I combined piecies of code from various projects
// I have added link to those below if you want to check them out
//
// I love JSON to CSV converter by Eric Mill, https://konklone.io/json/ 
// Source code of it is at https://github.com/konklone/json
// It does an awesome job of converting JSON to CSV in the front end
//
// I wanted to do the same in backend and the pipedream seemed like a good place
// To host that serverless code. at some point I will write a proper serverless code
// Where if I send the url of JSON it will return CSV. Until then this hack is good 
// enough for me.
//
// The final datawrapper graph is at https://datawrapper.dwcdn.net/ThC0E/2/
// It caches the data, so its not a major hit on pipedream.
//########################################################################################
const axios = require('axios')

//########################################################################################
// Pass in the objects to merge as arguments.
// For a deep extend, set the first argument to `true`.
// https://gomakethings.com/vanilla-javascript-version-of-jquery-extend/
var extend = function () {

    // Variables
    var extended = {};
    var deep = false;
    var i = 0;
    var length = arguments.length;

    // Check if a deep merge
    if ( Object.prototype.toString.call( arguments[0] ) === '[object Boolean]' ) {
        deep = arguments[0];
        i++;
    }

    // Merge the object into the extended object
    var merge = function (obj) {
        for ( var prop in obj ) {
            if ( Object.prototype.hasOwnProperty.call( obj, prop ) ) {
                // If deep merge and property is an object, merge properties
                if ( deep && Object.prototype.toString.call(obj[prop]) === '[object Object]' ) {
                    extended[prop] = extend( true, extended[prop], obj[prop] );
                } else {
                    extended[prop] = obj[prop];
                }
            }
        }
    };

    // Loop through each object and conduct a merge
    for ( ; i < length; i++ ) {
        var obj = arguments[i];
        merge(obj);
    }

    return extended;

};

//########################################################################################
//https://konklone.io/json/
//https://github.com/konklone/json/blob/gh-pages/assets/site.js
function parse_object(obj, path) {
    if (path == undefined)
        path = "";

    var type = typeof(obj);
    var scalar = (type == "number" || type == "string" || type == "boolean" || type == "null");

    if (type == "array" || type == "object") {
        var d = {};
        for (var i in obj) {

            var newD = parse_object(obj[i], path + i + "/");
            d = extend(d, newD);
        }

        return d;
    }

    else if (scalar) {
        var d = {};
        var endPath = path.substr(0, path.length-1);
        d[endPath] = obj;
        return d;
    }

    // ?
    else return {};
}

// otherwise, just find the first one
function arrayFrom(json) {
    var queue = [], next = json;
    while (next !== undefined) {
        if (typeof(next) == "array") {

            // but don't if it's just empty, or an array of scalars
            if (next.length > 0) {

              var type = typeof(next[0]);
              var scalar = (type == "number" || type == "string" || type == "boolean" || type == "null");

              if (!scalar)
                return next;
            }
        } if (typeof(next) == "object") {
          for (var key in next)
             queue.push(next[key]);
        }
        next = queue.shift();
    }
    // none found, consider the whole object a row
    return [json];
}

function removeTrailingComma(input) {
  if (input.slice(-1) == ",")
    return input.slice(0,-1);
  else
    return input;
}

function flatten(inArray) {
    var outArray = [];
    for (var row in inArray)
        outArray[outArray.length] = parse_object(inArray[row]);
    return outArray
}
//########################################################################################
//https://www.newline.co/@anthonygore/how-to-convert-json-to-csv-in-nodejs--23c6b226
function convert_to_csv(data){
    csv = data.map(row => Object.values(row));
    csv.unshift(Object.keys(data[0]));
    csv = csv.join('\n');
    console.log(csv)
    return csv
}
//########################################################################################
const dataSourceUrl = "https://raw.githubusercontent.com/datameet/covid19/master/data/mohfw_vaccination_status.json"
const data_json =  (await axios.get(dataSourceUrl)).data
const data = flatten(data_json.rows)
console.log(data)


$respond({
  immediate: true,
  status: 200,
  headers: {"Content-type":"text/plain"},
  body: convert_to_csv(data)
})