From 0bd68797795d595f7977b1e179022d9f4a375dea Mon Sep 17 00:00:00 2001 From: Hawken Rives Date: Sun, 5 May 2024 20:38:27 -0400 Subject: [PATCH 1/7] add dep on ical.js --- package-lock.json | 6 ++++++ package.json | 1 + 2 files changed, 7 insertions(+) diff --git a/package-lock.json b/package-lock.json index 889c0a74..935635a1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "dotenv": "16.4.5", "get-urls": "12.1.0", "html-entities": "2.5.2", + "ical.js": "^2.0.1", "is-absolute-url": "4.0.1", "jsdom": "24.0.0", "koa": "2.15.3", @@ -1310,6 +1311,11 @@ "resolved": "https://registry.npmjs.org/humanize-number/-/humanize-number-0.0.2.tgz", "integrity": "sha512-un3ZAcNQGI7RzaWGZzQDH47HETM4Wrj6z6E4TId8Yeq9w5ZKUVB1nrT2jwFheTUjEmqcgTjXDc959jum+ai1kQ==" }, + "node_modules/ical.js": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ical.js/-/ical.js-2.0.1.tgz", + "integrity": "sha512-uYYb1CwTXbd9NP/xTtgQZ5ivv6bpUjQu9VM98s3X78L3XRu00uJW5ZtmnLwyxhztpf5fSiRyDpFW7ZNCePlaPw==" + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", diff --git a/package.json b/package.json index 8aa055c3..1445ebc9 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "dotenv": "16.4.5", "get-urls": "12.1.0", "html-entities": "2.5.2", + "ical.js": "^2.0.1", "is-absolute-url": "4.0.1", "jsdom": "24.0.0", "koa": "2.15.3", From d28dec891da8568425aae30703174f45ff2ffa06 Mon Sep 17 00:00:00 2001 From: Hawken Rives Date: Sun, 5 May 2024 20:38:40 -0400 Subject: [PATCH 2/7] fix folder for presence --- source/ccci-stolaf-college/v1/orgs.js | 2 +- source/{calendar-presence/index.js => student-orgs/presence.js} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename source/{calendar-presence/index.js => student-orgs/presence.js} (100%) diff --git a/source/ccci-stolaf-college/v1/orgs.js b/source/ccci-stolaf-college/v1/orgs.js index 2db68cbe..e096eca2 100644 --- a/source/ccci-stolaf-college/v1/orgs.js +++ b/source/ccci-stolaf-college/v1/orgs.js @@ -1,7 +1,7 @@ import {ONE_HOUR} from '../../ccc-lib/constants.js' import mem from 'memoize' -import {presence as _presence} from '../../calendar-presence/index.js' +import {presence as _presence} from '../../student-orgs/presence.js' const CACHE_DURATION = ONE_HOUR * 36 diff --git a/source/calendar-presence/index.js b/source/student-orgs/presence.js similarity index 100% rename from source/calendar-presence/index.js rename to source/student-orgs/presence.js From 1d051dfed532288f5c79a3d61260ef159986687a Mon Sep 17 00:00:00 2001 From: Hawken Rives Date: Sun, 5 May 2024 20:39:11 -0400 Subject: [PATCH 3/7] relocate google+reason calendar sections; add Zod to ensure output types --- .../index.js => calendar/google.js} | 13 ++++++------- .../index.js => calendar/reason.js} | 6 ++++-- source/calendar/types.js | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 9 deletions(-) rename source/{calendar-google/index.js => calendar/google.js} (87%) rename source/{calendar-reason/index.js => calendar/reason.js} (98%) create mode 100644 source/calendar/types.js diff --git a/source/calendar-google/index.js b/source/calendar/google.js similarity index 87% rename from source/calendar-google/index.js rename to source/calendar/google.js index 50f49385..d804ccf0 100644 --- a/source/calendar-google/index.js +++ b/source/calendar/google.js @@ -2,18 +2,19 @@ import {get} from '../ccc-lib/http.js' import moment from 'moment' import getUrls from 'get-urls' import {JSDOM} from 'jsdom' +import {Event} from './types.js' function convertGoogleEvents(data, now = moment()) { - let events = data.map((event) => { + return data.map((event) => { const startTime = moment(event.start.date || event.start.dateTime) const endTime = moment(event.end.date || event.end.dateTime) let description = (event.description || '').replace('
', '\n') description = JSDOM.fragment(description).textContent.trim() - return { + return Event.parse({ dataSource: 'google', - startTime, - endTime, + startTime: startTime.toISOString(), + endTime: endTime.toISOString(), title: event.summary || '', description: description, location: event.location || '', @@ -24,10 +25,8 @@ function convertGoogleEvents(data, now = moment()) { endTime: true, subtitle: 'location', }, - } + }) }) - - return events } export async function googleCalendar(calendarId, now = moment()) { diff --git a/source/calendar-reason/index.js b/source/calendar/reason.js similarity index 98% rename from source/calendar-reason/index.js rename to source/calendar/reason.js index af83e880..12874658 100644 --- a/source/calendar-reason/index.js +++ b/source/calendar/reason.js @@ -5,6 +5,8 @@ import moment from 'moment-timezone' import lodash from 'lodash' import getUrls from 'get-urls' import {JSDOM} from 'jsdom' +import {Event} from './types.js' + const {dropWhile, dropRightWhile, sortBy} = lodash const TZ = 'US/Central' @@ -116,7 +118,7 @@ function convertReasonEvent(event, now = moment()) { let links = description ? [...getUrls(description)] : [] - return { + return Event.parse({ dataSource: 'reason', startTime: event.startTime, endTime: event.endTime, @@ -133,7 +135,7 @@ function convertReasonEvent(event, now = moment()) { endTime: true, subtitle: 'location', }, - } + }) } export async function reasonCalendar(calendarUrl, now = moment()) { diff --git a/source/calendar/types.js b/source/calendar/types.js new file mode 100644 index 00000000..81ee8a07 --- /dev/null +++ b/source/calendar/types.js @@ -0,0 +1,19 @@ +import {z} from 'zod' + +const EventConfig = z.object({ + startTime: z.boolean(), + endTime: z.boolean(), + subtitle: z.union([z.literal('location')]), +}) + +export const Event = z.object({ + dataSource: z.string(), + startTime: z.string().datetime(), + endTime: z.string().datetime(), + title: z.string(), + description: z.string(), + isOngoing: z.boolean(), + links: z.array(z.unknown()), + config: EventConfig, + metadata: z.optional(z.unknown()), +}) From a9d13897eae4320fc44da2023ac6f02d1128253e Mon Sep 17 00:00:00 2001 From: Hawken Rives Date: Sun, 5 May 2024 20:39:28 -0400 Subject: [PATCH 4/7] add ical parsing module --- source/calendar/ical.js | 49 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 source/calendar/ical.js diff --git a/source/calendar/ical.js b/source/calendar/ical.js new file mode 100644 index 00000000..f8bb21c3 --- /dev/null +++ b/source/calendar/ical.js @@ -0,0 +1,49 @@ +import {get} from '../ccc-lib/http.js' +import moment from 'moment' +import getUrls from 'get-urls' +import {JSDOM} from 'jsdom' +import InternetCalendar from 'ical.js' +import {Event} from './types.js' + +/** + * @param {InternetCalendar.Event[]} data + * @param {typeof moment} now + * @returns {Event[]} + */ +function convertEvents(data, now = moment()) { + return data.map((event) => { + const startTime = moment(event.startDate.toString()) + const endTime = moment(event.endDate.toString()) + let description = JSDOM.fragment(event.description || '').textContent.trim() + + return Event.parse({ + dataSource: 'ical', + startTime: startTime.toISOString(), + endTime: endTime.toISOString(), + title: event.summary, + description: description, + location: event.location, + isOngoing: startTime.isBefore(now, 'day'), + links: [...getUrls(description)], + metadata: { + uid: event.uid, + }, + config: { + startTime: true, + endTime: true, + subtitle: 'location', + }, + }) + }) +} + +export async function ical(url, now = moment()) { + let body = await get(url).text() + + let comp = InternetCalendar.Component.fromString(body) + let events = comp + .getAllSubcomponents('vevent') + .map((vevent) => new InternetCalendar.Event(vevent)) + + return convertEvents(events, now) +} From 71934c65a46b0b38e3f2c79d91299e2cd1fdb46b Mon Sep 17 00:00:00 2001 From: Hawken Rives Date: Sun, 5 May 2024 20:39:36 -0400 Subject: [PATCH 5/7] use ical parsing module for carleton calendars --- source/ccci-carleton-college/v1/calendar.js | 32 +++++++++------------ source/ccci-carleton-college/v1/index.js | 1 - source/ccci-stolaf-college/v1/calendar.js | 14 ++++----- source/ccci-stolaf-college/v1/index.js | 1 - 4 files changed, 18 insertions(+), 30 deletions(-) diff --git a/source/ccci-carleton-college/v1/calendar.js b/source/ccci-carleton-college/v1/calendar.js index 19b5e489..d21784d0 100644 --- a/source/ccci-carleton-college/v1/calendar.js +++ b/source/ccci-carleton-college/v1/calendar.js @@ -1,10 +1,10 @@ -import {googleCalendar} from '../../calendar-google/index.js' -import {reasonCalendar} from '../../calendar-reason/index.js' +import {googleCalendar} from '../../calendar/google.js' +import {ical} from '../../calendar/ical.js' import {ONE_MINUTE} from '../../ccc-lib/constants.js' import mem from 'memoize' export const getGoogleCalendar = mem(googleCalendar, {maxAge: ONE_MINUTE}) -export const getReasonCalendar = mem(reasonCalendar, {maxAge: ONE_MINUTE}) +export const getInternetCalendar = mem(ical, {maxAge: ONE_MINUTE}) export async function google(ctx) { ctx.cacheControl(ONE_MINUTE) @@ -13,33 +13,27 @@ export async function google(ctx) { ctx.body = await getGoogleCalendar(calendarId) } -export async function reason(ctx) { +export async function ics(ctx) { ctx.cacheControl(ONE_MINUTE) let {url: calendarUrl} = ctx.query - ctx.body = await getReasonCalendar(calendarUrl) -} - -export function ics(ctx) { - ctx.cacheControl(ONE_MINUTE) - - ctx.throw(501, 'ICS support is not implemented yet.') + ctx.body = await getInternetCalendar(calendarUrl) } export async function carleton(ctx) { ctx.cacheControl(ONE_MINUTE) let url = - 'webcal://www.carleton.edu/calendar/?loadFeed=calendar&stamp=1714843628' - ctx.body = await getGoogleCalendar(url) + 'https://www.carleton.edu/calendar/?loadFeed=calendar&stamp=1714843628' + ctx.body = await getInternetCalendar(url) } export async function cave(ctx) { ctx.cacheControl(ONE_MINUTE) let url = - 'webcal://www.carleton.edu/student/orgs/cave/calendar/?loadFeed=calendar&stamp=1714844429\n' - ctx.body = await getGoogleCalendar(url) + 'https://www.carleton.edu/student/orgs/cave/calendar/?loadFeed=calendar&stamp=1714844429\n' + ctx.body = await getInternetCalendar(url) } export async function stolaf(ctx) { @@ -74,14 +68,14 @@ export async function convos(ctx) { ctx.cacheControl(ONE_MINUTE) let url = - 'webcal://www.carleton.edu/convocations/calendar/?loadFeed=calendar&stamp=1714843936' - ctx.body = await getGoogleCalendar(url) + 'https://www.carleton.edu/convocations/calendar/?loadFeed=calendar&stamp=1714843936' + ctx.body = await getInternetCalendar(url) } export async function sumo(ctx) { ctx.cacheControl(ONE_MINUTE) let url = - 'webcal://www.carleton.edu/student/orgs/sumo/schedule/?loadFeed=calendar&stamp=1714840383' - ctx.body = await getGoogleCalendar(url) + 'https://www.carleton.edu/student/orgs/sumo/schedule/?loadFeed=calendar&stamp=1714840383' + ctx.body = await getInternetCalendar(url) } diff --git a/source/ccci-carleton-college/v1/index.js b/source/ccci-carleton-college/v1/index.js index 20928a1d..00b479a5 100644 --- a/source/ccci-carleton-college/v1/index.js +++ b/source/ccci-carleton-college/v1/index.js @@ -53,7 +53,6 @@ api.get('/food/named/menu/schulze', menus.schulzeMenu) // calendar api.get('/calendar/google', calendar.google) -api.get('/calendar/reason', calendar.reason) api.get('/calendar/ics', calendar.ics) api.get('/calendar/named/carleton', calendar.carleton) api.get('/calendar/named/the-cave', calendar.cave) diff --git a/source/ccci-stolaf-college/v1/calendar.js b/source/ccci-stolaf-college/v1/calendar.js index 82c907bc..8dddac52 100644 --- a/source/ccci-stolaf-college/v1/calendar.js +++ b/source/ccci-stolaf-college/v1/calendar.js @@ -1,10 +1,10 @@ -import {googleCalendar} from '../../calendar-google/index.js' -import {reasonCalendar} from '../../calendar-reason/index.js' +import {googleCalendar} from '../../calendar/google.js' +import {ical} from '../../calendar/ical.js' import {ONE_MINUTE} from '../../ccc-lib/constants.js' import mem from 'memoize' export const getGoogleCalendar = mem(googleCalendar, {maxAge: ONE_MINUTE}) -export const getReasonCalendar = mem(reasonCalendar, {maxAge: ONE_MINUTE}) +export const getInternetCalendar = mem(ical, {maxAge: ONE_MINUTE}) export async function google(ctx) { ctx.cacheControl(ONE_MINUTE) @@ -13,15 +13,11 @@ export async function google(ctx) { ctx.body = await getGoogleCalendar(calendarId) } -export async function reason(ctx) { +export async function ics(ctx) { ctx.cacheControl(ONE_MINUTE) let {url: calendarUrl} = ctx.query - ctx.body = await getReasonCalendar(calendarUrl) -} - -export function ics(ctx) { - ctx.throw(501, 'ICS support is not implemented yet.') + ctx.body = await getInternetCalendar(calendarUrl) } export async function stolaf(ctx) { diff --git a/source/ccci-stolaf-college/v1/index.js b/source/ccci-stolaf-college/v1/index.js index 62a2d470..b888fa0e 100644 --- a/source/ccci-stolaf-college/v1/index.js +++ b/source/ccci-stolaf-college/v1/index.js @@ -56,7 +56,6 @@ api.get('/food/named/menu/schulze', menus.schulzeMenu) // calendar api.get('/calendar/google', calendar.google) -api.get('/calendar/reason', calendar.reason) api.get('/calendar/ics', calendar.ics) api.get('/calendar/named/stolaf', calendar.stolaf) api.get('/calendar/named/oleville', calendar.oleville) From c39a493e98257be28687ea140fe7c7253f41690a Mon Sep 17 00:00:00 2001 From: Hawken Rives Date: Sun, 5 May 2024 20:52:43 -0400 Subject: [PATCH 6/7] exclude past events from ics endpoints --- source/calendar/ical.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/source/calendar/ical.js b/source/calendar/ical.js index f8bb21c3..4f4205bf 100644 --- a/source/calendar/ical.js +++ b/source/calendar/ical.js @@ -4,6 +4,9 @@ import getUrls from 'get-urls' import {JSDOM} from 'jsdom' import InternetCalendar from 'ical.js' import {Event} from './types.js' +import lodash from 'lodash' + +const {sortBy} = lodash /** * @param {InternetCalendar.Event[]} data @@ -37,7 +40,7 @@ function convertEvents(data, now = moment()) { }) } -export async function ical(url, now = moment()) { +export async function ical(url, {onlyFuture = true} = {}, now = moment()) { let body = await get(url).text() let comp = InternetCalendar.Component.fromString(body) @@ -45,5 +48,11 @@ export async function ical(url, now = moment()) { .getAllSubcomponents('vevent') .map((vevent) => new InternetCalendar.Event(vevent)) - return convertEvents(events, now) + if (onlyFuture) { + events = events.filter((event) => + moment(event.endDate.toString()).isAfter(now, 'day'), + ) + } + + return sortBy(convertEvents(events, now), (event) => event.startTime) } From e1601683fe4cdd287af6c87435b4d427c71e8ada Mon Sep 17 00:00:00 2001 From: Hawken Rives Date: Sun, 5 May 2024 20:55:53 -0400 Subject: [PATCH 7/7] move all endpoints to ics --- source/ccci-carleton-college/v1/calendar.js | 4 +-- source/ccci-stolaf-college/v1/calendar.js | 32 ++++++++++++++------- source/ccci-stolaf-college/v1/index.js | 1 + 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/source/ccci-carleton-college/v1/calendar.js b/source/ccci-carleton-college/v1/calendar.js index d21784d0..6f386772 100644 --- a/source/ccci-carleton-college/v1/calendar.js +++ b/source/ccci-carleton-college/v1/calendar.js @@ -39,8 +39,8 @@ export async function cave(ctx) { export async function stolaf(ctx) { ctx.cacheControl(ONE_MINUTE) - let id = '5g91il39n0sv4c2bjdv1jrvcpq4ulm4r@import.calendar.google.com' - ctx.body = await getGoogleCalendar(id) + let id = 'https://www.stolaf.edu/apps/calendar/ical.cfm' + ctx.body = await getInternetCalendar(id) } export async function northfield(ctx) { diff --git a/source/ccci-stolaf-college/v1/calendar.js b/source/ccci-stolaf-college/v1/calendar.js index 8dddac52..9d0af936 100644 --- a/source/ccci-stolaf-college/v1/calendar.js +++ b/source/ccci-stolaf-college/v1/calendar.js @@ -23,34 +23,46 @@ export async function ics(ctx) { export async function stolaf(ctx) { ctx.cacheControl(ONE_MINUTE) - let id = '5g91il39n0sv4c2bjdv1jrvcpq4ulm4r@import.calendar.google.com' - ctx.body = await getGoogleCalendar(id) + let id = 'https://www.stolaf.edu/apps/calendar/ical.cfm' + ctx.body = await getInternetCalendar(id) } export async function oleville(ctx) { ctx.cacheControl(ONE_MINUTE) - let id = 'opha089fhthpchc0pkdqinca44nl7svk@import.calendar.google.com' - ctx.body = await getGoogleCalendar(id) + let id = + 'https://calendar.google.com/calendar/ical/opha089fhthpchc0pkdqinca44nl7svk%40import.calendar.google.com/public/basic.ics' + ctx.body = await getInternetCalendar(id) +} + +export async function thePause(ctx) { + ctx.cacheControl(ONE_MINUTE) + + let id = + 'https://calendar.google.com/calendar/ical/stolaf.edu_qkrej5rm8c8582dlnc28nreboc%40group.calendar.google.com/public/basic.ics' + ctx.body = await getInternetCalendar(id) } export async function northfield(ctx) { ctx.cacheControl(ONE_MINUTE) - let id = 'thisisnorthfield@gmail.com' - ctx.body = await getGoogleCalendar(id) + let id = + 'https://calendar.google.com/calendar/ical/thisisnorthfield%40gmail.com/public/basic.ics' + ctx.body = await getInternetCalendar(id) } export async function krlx(ctx) { ctx.cacheControl(ONE_MINUTE) - let id = 'krlxradio88.1@gmail.com' - ctx.body = await getGoogleCalendar(id) + let id = + 'https://calendar.google.com/calendar/ical/krlxradio88.1%40gmail.com/public/basic.ics' + ctx.body = await getInternetCalendar(id) } export async function ksto(ctx) { ctx.cacheControl(ONE_MINUTE) - let id = 'stolaf.edu_7u3lgo4rr3o9dchr50q982ribk@group.calendar.google.com' - ctx.body = await getGoogleCalendar(id) + let id = + 'https://calendar.google.com/calendar/ical/stolaf.edu_7u3lgo4rr3o9dchr50q982ribk%40group.calendar.google.com/public/basic.ics' + ctx.body = await getInternetCalendar(id) } diff --git a/source/ccci-stolaf-college/v1/index.js b/source/ccci-stolaf-college/v1/index.js index b888fa0e..7e06f7a1 100644 --- a/source/ccci-stolaf-college/v1/index.js +++ b/source/ccci-stolaf-college/v1/index.js @@ -59,6 +59,7 @@ api.get('/calendar/google', calendar.google) api.get('/calendar/ics', calendar.ics) api.get('/calendar/named/stolaf', calendar.stolaf) api.get('/calendar/named/oleville', calendar.oleville) +api.get('/calendar/named/the-pause', calendar.thePause) api.get('/calendar/named/northfield', calendar.northfield) api.get('/calendar/named/krlx-schedule', calendar.krlx) api.get('/calendar/named/ksto-schedule', calendar.ksto)