You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
75 lines
2.4 KiB
TypeScript
75 lines
2.4 KiB
TypeScript
import { parse as tomlParse, stringify as tomlStringify } from "https://deno.land/std@0.130.0/encoding/toml.ts"
|
|
|
|
const reFrontmatter = /^\+\+\+([\s\S]*)^\+\+\+$([\s\S]*)/m;
|
|
|
|
export function url2title (url: string): string {
|
|
return url
|
|
.replace(/^https?:\/\//, '') // remove leading http(s)://
|
|
.replace(/\/$/, '') // remove trailing slash
|
|
}
|
|
|
|
// gets an URL like https://foo.bar and returns ./content/foo_baz.md
|
|
function url2filepath (url: string, output_path: string): string {
|
|
const filename = url2title(url)
|
|
.replaceAll(/[\.\/]/g, '_') // replace dots and slashes with underscores
|
|
return `${output_path}/${filename}.md`
|
|
}
|
|
|
|
// deprecated in deno std, but also simple to replicate
|
|
// see: https://deno.land/std@0.130.0/fs/exists.ts
|
|
async function exists (path: string): Promise<boolean> {
|
|
try {
|
|
return !!(await Deno.lstat(path))
|
|
} catch (err) {
|
|
if (err instanceof Deno.errors.NotFound) return false
|
|
throw err
|
|
}
|
|
}
|
|
|
|
// checks if URL has a record already and returns time since last check or null
|
|
export async function getPageRecord (url: string, output_path: string): Promise<PageRecord|null> {
|
|
const path = url2filepath(url, output_path)
|
|
const hasRecord = await exists(path)
|
|
|
|
if (!hasRecord) return null
|
|
|
|
const fileContents = await Deno.readTextFile(path)
|
|
const match = fileContents.match(reFrontmatter)
|
|
if (!match) return null // that should never happen but who knows
|
|
return tomlParse(match[1].trim()) as PageRecord
|
|
}
|
|
|
|
export async function writeRecord (record: PageRecord, url: string, output_path: string): Promise<boolean> {
|
|
const path = url2filepath(url, output_path)
|
|
const toml = tomlStringify(record)
|
|
|
|
try {
|
|
await Deno.writeTextFile(path, `+++\n${toml}+++\n`)
|
|
return true
|
|
} catch {
|
|
return false
|
|
}
|
|
}
|
|
|
|
function delay (ms: number): Promise<unknown> {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
export async function retryFetch (url: string, retries=3, msDelay=1000): Promise<any> {
|
|
try {
|
|
const response = await fetch(url)
|
|
if (!response.ok) return false
|
|
const json = await response.json()
|
|
return json
|
|
} catch (err) {
|
|
if (retries > 0) {
|
|
console.warn(`Failed to fetch ${url}, retrying in ${msDelay}ms.`)
|
|
await delay(msDelay)
|
|
return retryFetch(url, retries - 1, msDelay)
|
|
} else {
|
|
console.error(`Fetching ${url} failed too often. Giving up.`)
|
|
return false
|
|
}
|
|
}
|
|
}
|