🎉 Pipedream raises $20M Series A 🎉
Read the blog post and TC Techcrunch article.
STREET CLEANING #1 - Get street cleaning schedules for a (lat, long)
@dylan
code:
data:privatelast updated:1 year ago
today
Build integrations remarkably fast!
You're viewing a public workflow template.
Sign up to customize, add steps, modify code and more.
Join 250,000+ developers using the Pipedream platform
steps.
trigger
HTTP API
Deploy to generate unique URL
This workflow runs on Pipedream's servers and is triggered by HTTP / Webhook requests.
steps.
confirm_lat_long
auth
to use OAuth tokens and API keys in code via theauths object
code
Write any Node.jscodeand use anynpm package. You can alsoexport datafor use in later steps via return or this.key = 'value', pass input data to your code viaparams, and maintain state across invocations with$checkpoint.
async (event, steps) => {
1
2
3
4
}
5
if (!event.body.lat || !event.body.long) {
  $end("Please pass both a lat and long to this workflow")
}
if (!event.body.lat || !event.body.long) {
  $end("Please pass both a lat and long to this workflow")
}
steps.
calculate_box_around_point
auth
to use OAuth tokens and API keys in code via theauths object
code
Write any Node.jscodeand use anynpm package. You can alsoexport datafor use in later steps via return or this.key = 'value', pass input data to your code viaparams, and maintain state across invocations with$checkpoint.
async (event, steps) => {
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
}
21
// To perform intersection queries with the SF Data API below,
// we need to create a box around the point where the recording was taken,
// and ask if any of our street segments (LineStrings) intersect with it
// We create a box with corners roughly 10 meters in every cardinal 
// direction from the original point

const lat = parseFloat(event.body.lat)
const long = parseFloat(event.body.long)
const DISTANCE_IN_EACH_DIRECTION = .0002

// Simple math to generate box around the point (for San Francisco)
this.topLeftCorner = [lat + DISTANCE_IN_EACH_DIRECTION, long - DISTANCE_IN_EACH_DIRECTION]
this.topRightCorner = [lat + DISTANCE_IN_EACH_DIRECTION, long + DISTANCE_IN_EACH_DIRECTION]
this.bottomRightCorner = [lat - DISTANCE_IN_EACH_DIRECTION, long + DISTANCE_IN_EACH_DIRECTION]
this.bottomLeftCorner = [lat - DISTANCE_IN_EACH_DIRECTION, long - DISTANCE_IN_EACH_DIRECTION]

// Generate WKT (well-known text represenation of a geometry) POLYGON
// that corresponds to the box. WKT orders coordinates as (long, lat)
this.POLYGON = `POLYGON((${this.topLeftCorner[1]} ${this.topLeftCorner[0]},${this.topRightCorner[1]} ${this.topRightCorner[0]},${this.bottomRightCorner[1]} ${this.bottomRightCorner[0]},${this.bottomLeftCorner[1]} ${this.bottomLeftCorner[0]},${this.topLeftCorner[1]} ${this.topLeftCorner[0]}))`
// To perform intersection queries with the SF Data API below,
// we need to create a box around the point where the recording was taken,
// and ask if any of our street segments (LineStrings) intersect with it
// We create a box with corners roughly 10 meters in every cardinal 
// direction from the original point

const lat = parseFloat(event.body.lat)
const long = parseFloat(event.body.long)
const DISTANCE_IN_EACH_DIRECTION = .0002

// Simple math to generate box around the point (for San Francisco)
this.topLeftCorner = [lat + DISTANCE_IN_EACH_DIRECTION, long - DISTANCE_IN_EACH_DIRECTION]
this.topRightCorner = [lat + DISTANCE_IN_EACH_DIRECTION, long + DISTANCE_IN_EACH_DIRECTION]
this.bottomRightCorner = [lat - DISTANCE_IN_EACH_DIRECTION, long + DISTANCE_IN_EACH_DIRECTION]
this.bottomLeftCorner = [lat - DISTANCE_IN_EACH_DIRECTION, long - DISTANCE_IN_EACH_DIRECTION]

// Generate WKT (well-known text represenation of a geometry) POLYGON
// that corresponds to the box. WKT orders coordinates as (long, lat)
this.POLYGON = `POLYGON((${this.topLeftCorner[1]} ${this.topLeftCorner[0]},${this.topRightCorner[1]} ${this.topRightCorner[0]},${this.bottomRightCorner[1]} ${this.bottomRightCorner[0]},${this.bottomLeftCorner[1]} ${this.bottomLeftCorner[0]},${this.topLeftCorner[1]} ${this.topLeftCorner[0]}))`
steps.
get_streets_within_bounding_box
auth
to use OAuth tokens and API keys in code via theauths object
(auths.san_francisco_open_data_datasf)
code
Write any Node.jscodeand use anynpm package. You can alsoexport datafor use in later steps via return or this.key = 'value', pass input data to your code viaparams, and maintain state across invocations with$checkpoint.
async (event, steps, auths) => {
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
}
16
// Street Sweeping schedules, by street
// https://data.sfgov.org/City-Infrastructure/Street-Sweeping-Schedule/yhqp-riqs

this.intersectionString = `intersects(line, '${steps.calculate_box_around_point.POLYGON}')`

return await require("@pipedreamhq/platform").axios(this, {
  url: `https://data.sfgov.org/resource/yhqp-riqs`,
  headers: {
    "X-App-Token": `${auths.san_francisco_open_data_datasf.app_token}`,
  },
  params: {
    "$where": this.intersectionString
  },
})
// Street Sweeping schedules, by street
// https://data.sfgov.org/City-Infrastructure/Street-Sweeping-Schedule/yhqp-riqs

this.intersectionString = `intersects(line, '${steps.calculate_box_around_point.POLYGON}')`

return await require("@pipedreamhq/platform").axios(this, {
  url: `https://data.sfgov.org/resource/yhqp-riqs`,
  headers: {
    "X-App-Token": `${auths.san_francisco_open_data_datasf.app_token}`,
  },
  params: {
    "$where": this.intersectionString
  },
})
steps.
format_street_cleaning_strings
auth
to use OAuth tokens and API keys in code via theauths object
code
Write any Node.jscodeand use anynpm package. You can alsoexport datafor use in later steps via return or this.key = 'value', pass input data to your code viaparams, and maintain state across invocations with$checkpoint.
async (event, steps) => {
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
}
106
// For each street within our bounding box, generate a
// string in the format of our street cleaning signs:
//
//   ${START_TIME} to ${END_TIME} ${WEEKDAY(S)}
//
// First, DataSF returns an entry per (street segment, side of street, cleaning day).
// Segments are referred to by Centerline Network Number. See
// https://data.sfgov.org/City-Infrastructure/Street-Sweeping-Schedule/yhqp-riqs .
// If a single side of a street is cleaned on Monday and Thursday, DataSF returns two
// entries. We need coalesce these entries into a single string.
// 
// Moreover, times returned from DataSF are in 24-hour clock time,
// But the time on signs are in 12-hour time, so we convert.

const capitalize = require("lodash.capitalize")
const includes = require("lodash.includes")
const uniq = require('lodash.uniq')

this.cleaningTimes = {}

const matchingStreets = steps.get_streets_within_bounding_box.$return_value
if (!matchingStreets.length) {
  $end("No streets returned from DataSF within bounding box")
}

for (const record of matchingStreets) {
  const { cnn, cnnrightleft, fromhour, tohour, weekday } = record

  // Street segment : side of street
  const key = `${cnn}:${cnnrightleft}`

  const cleaningTimeInstance = `${convert24HourTimeTo12HourTime(fromhour)} to ${convert24HourTimeTo12HourTime(tohour)}`
  const weekdayLower = weekday.toLowerCase()

  // If we've already stored the time portion of the string, add the day to the end
  if (key in this.cleaningTimes) {
    if (!(includes(this.cleaningTimes[key]["weekdays"], weekdayLower))) {
      this.cleaningTimes[key]["weekdays"].push(weekdayLower)
      this.cleaningTimes[key]["instances"].push({ fromhour, tohour, weekday })
    }
    continue
  }

  // Otherwise, add the time / date of this record
  this.cleaningTimes[key] = {
    schedule: cleaningTimeInstance,
    weekdays: [weekdayLower],
    instances: [{ fromhour, tohour, weekday }]
  }
}

this.scheduleArray = []
this.instances = {}

for (const { schedule, weekdays, instances } of Object.values(this.cleaningTimes)) {
  const uniqueWeekdays = uniq(weekdays.sort(compareWeekdays)).map(capitalize)
  const scheduleString = `${schedule} ${uniqueWeekdays.join(", ")}`
  if (includes(this.scheduleArray, scheduleString)) {
    continue
  }
  this.scheduleArray.push(scheduleString)
  this.instances[scheduleString] = instances
}

// HTTP RESPONSE
$respond({
  body: { 
    schedules: this.scheduleArray,
    instances: this.instances,
  },
})

function convert24HourTimeTo12HourTime(stringHour) {
  const hour = parseInt(stringHour)
  if (hour >= 0 && hour <= 11) {
    return `${hour}AM`
  }
  if (hour === 12) {
    return `${hour}PM`
  }
  return `${hour - 12}PM`
}

function compareWeekdays(a, b) {
  const weekdayNum = {
    // Schedules contain abbreviated and full weekdays
    'sun': 0,
    'sunday': 0,
    'mon': 1,
    'monday': 1,
    'tues': 2,
    'tuesday': 2,
    'wed': 3,
    'wednesday': 3,
    'thu': 4,
    'thursday': 4,
    'fri': 6,
    'friday': 6,
    'sat': 7,
    'saturday': 7,
  }

  return weekdayNum[a.toLowerCase()] - weekdayNum[b.toLowerCase()]
}
// For each street within our bounding box, generate a
// string in the format of our street cleaning signs:
//
//   ${START_TIME} to ${END_TIME} ${WEEKDAY(S)}
//
// First, DataSF returns an entry per (street segment, side of street, cleaning day).
// Segments are referred to by Centerline Network Number. See
// https://data.sfgov.org/City-Infrastructure/Street-Sweeping-Schedule/yhqp-riqs .
// If a single side of a street is cleaned on Monday and Thursday, DataSF returns two
// entries. We need coalesce these entries into a single string.
// 
// Moreover, times returned from DataSF are in 24-hour clock time,
// But the time on signs are in 12-hour time, so we convert.

const capitalize = require("lodash.capitalize")
const includes = require("lodash.includes")
const uniq = require('lodash.uniq')

this.cleaningTimes = {}

const matchingStreets = steps.get_streets_within_bounding_box.$return_value
if (!matchingStreets.length) {
  $end("No streets returned from DataSF within bounding box")
}

for (const record of matchingStreets) {
  const { cnn, cnnrightleft, fromhour, tohour, weekday } = record

  // Street segment : side of street
  const key = `${cnn}:${cnnrightleft}`

  const cleaningTimeInstance = `${convert24HourTimeTo12HourTime(fromhour)} to ${convert24HourTimeTo12HourTime(tohour)}`
  const weekdayLower = weekday.toLowerCase()

  // If we've already stored the time portion of the string, add the day to the end
  if (key in this.cleaningTimes) {
    if (!(includes(this.cleaningTimes[key]["weekdays"], weekdayLower))) {
      this.cleaningTimes[key]["weekdays"].push(weekdayLower)
      this.cleaningTimes[key]["instances"].push({ fromhour, tohour, weekday })
    }
    continue
  }

  // Otherwise, add the time / date of this record
  this.cleaningTimes[key] = {
    schedule: cleaningTimeInstance,
    weekdays: [weekdayLower],
    instances: [{ fromhour, tohour, weekday }]
  }
}

this.scheduleArray = []
this.instances = {}

for (const { schedule, weekdays, instances } of Object.values(this.cleaningTimes)) {
  const uniqueWeekdays = uniq(weekdays.sort(compareWeekdays)).map(capitalize)
  const scheduleString = `${schedule} ${uniqueWeekdays.join(", ")}`
  if (includes(this.scheduleArray, scheduleString)) {
    continue
  }
  this.scheduleArray.push(scheduleString)
  this.instances[scheduleString] = instances
}

// HTTP RESPONSE
$respond({
  body: { 
    schedules: this.scheduleArray,
    instances: this.instances,
  },
})

function convert24HourTimeTo12HourTime(stringHour) {
  const hour = parseInt(stringHour)
  if (hour >= 0 && hour <= 11) {
    return `${hour}AM`
  }
  if (hour === 12) {
    return `${hour}PM`
  }
  return `${hour - 12}PM`
}

function compareWeekdays(a, b) {
  const weekdayNum = {
    // Schedules contain abbreviated and full weekdays
    'sun': 0,
    'sunday': 0,
    'mon': 1,
    'monday': 1,
    'tues': 2,
    'tuesday': 2,
    'wed': 3,
    'wednesday': 3,
    'thu': 4,
    'thursday': 4,
    'fri': 6,
    'friday': 6,
    'sat': 7,
    'saturday': 7,
  }

  return weekdayNum[a.toLowerCase()] - weekdayNum[b.toLowerCase()]
}