diff options
25 files changed, 1008 insertions, 765 deletions
diff --git a/deno/mail-relay/.gitignore b/deno/.gitignore index 327aef0..327aef0 100644 --- a/deno/mail-relay/.gitignore +++ b/deno/.gitignore diff --git a/deno/base/config.ts b/deno/base/config.ts new file mode 100644 index 0000000..98722f6 --- /dev/null +++ b/deno/base/config.ts @@ -0,0 +1,94 @@ +import { camelCaseToKebabCase } from "./text.ts"; + +export interface ConfigDefinitionItem { + readonly description: string; + readonly default?: string; + readonly secret?: boolean; +} + +interface ConfigMapItem extends ConfigDefinitionItem { + readonly env: string; + value?: string; +} + +export type ConfigDefinition<K extends string = string> = Record< + K, + ConfigDefinitionItem +>; +type ConfigMap<K extends string = string> = Record<K, ConfigMapItem>; + +export class ConfigProvider<K extends string> { + readonly #prefix: string; + readonly #map: ConfigMap<K>; + + constructor(prefix: string, ...definitions: Partial<ConfigDefinition<K>>[]) { + this.#prefix = prefix; + + const map: ConfigMap = {}; + for (const definition of definitions) { + for (const [key, def] of Object.entries(definition as ConfigDefinition)) { + map[key] = { + ...def, + env: `${this.#prefix}-${camelCaseToKebabCase(key as string)}` + .replaceAll("-", "_") + .toUpperCase(), + }; + } + } + this.#map = map as ConfigMap<K>; + } + + resolveFromEnv(options?: { keys?: K[] }) { + const keys = options?.keys ?? Object.keys(this.#map); + for (const key of keys) { + const { env, description, default: _default } = this.#map[key as K]; + const value = Deno.env.get(env) ?? _default; + if (value == null) { + throw new Error(`Required env ${env} (${description}) is not set.`); + } + this.#map[key as K].value = value; + } + } + + get(key: K): string { + if (!(key in this.#map)) { + throw new Error(`Unknown config key ${key as string}.`); + } + if (this.#map[key].value == null) { + this.resolveFromEnv({ keys: [key] }); + } + return this.#map[key].value!; + } + + set(key: K, value: string) { + if (!(key in this.#map)) { + throw new Error(`Unknown config key ${key as string}.`); + } + this.#map[key].value = value; + } + + getInt(key: K): number { + return Number(this.get(key)); + } + + getList(key: K, separator: string = ","): string[] { + const value = this.get(key); + if (value.length === 0) return []; + return value.split(separator); + } + + [Symbol.for("Deno.customInspect")]() { + const getValueString = (item: ConfigMapItem): string => { + if (item.value == null) return "(unresolved)"; + if (item.secret === true) return "***"; + return item.value; + }; + + return Object.entries(this.#map as ConfigMap) + .map( + ([key, item]) => `${key} [env: ${item.env}]: ${getValueString(item)}`, + ) + .join("\n"); + } +} + diff --git a/deno/mail-relay/cron.ts b/deno/base/cron.ts index bf0a0be..bf0a0be 100644 --- a/deno/mail-relay/cron.ts +++ b/deno/base/cron.ts diff --git a/deno/base/date.ts b/deno/base/date.ts new file mode 100644 index 0000000..e65691e --- /dev/null +++ b/deno/base/date.ts @@ -0,0 +1,6 @@ +export function toFileNameString(date: Date, dateOnly?: boolean): string { + const str = date.toISOString(); + return dateOnly === true + ? str.slice(0, str.indexOf("T")) + : str.replaceAll(/:|\./g, "-"); +} diff --git a/deno/base/deno.json b/deno/base/deno.json new file mode 100644 index 0000000..2c2d550 --- /dev/null +++ b/deno/base/deno.json @@ -0,0 +1,11 @@ +{ + "name": "@crupest/base", + "version": "0.1.0", + "exports": { + "./config": "./config.ts", + "./cron": "./cron.ts", + "./date": "./date.ts", + "./text": "./text.ts", + "./log": "./log.ts" + } +} diff --git a/deno/base/log.ts b/deno/base/log.ts new file mode 100644 index 0000000..1a4942d --- /dev/null +++ b/deno/base/log.ts @@ -0,0 +1,164 @@ +import { join } from "@std/path"; + +import { toFileNameString } from "./date.ts"; + +export type LogLevel = "error" | "warn" | "info"; + +export interface LogEntry { + content: [unknown, ...unknown[]]; + level?: LogLevel; + cause?: unknown; +} + +export interface LogEntryBuilder { + withLevel(level: LogLevel): LogEntryBuilder; + withCause(cause: unknown): LogEntryBuilder; + setError(error: boolean): LogEntryBuilder; + write(): void; +} + +export interface ExternalLogStream extends Disposable { + stream: WritableStream; +} + +export class Logger { + #indentSize = 2; + #externalLogDir?: string; + + #contextStack: { depth: number; level: LogLevel }[] = [ + { depth: 0, level: "info" }, + ]; + + get #context() { + return this.#contextStack.at(-1)!; + } + + get indentSize() { + return this.#indentSize; + } + + set indentSize(value: number) { + this.#indentSize = value; + } + + get externalLogDir() { + return this.#externalLogDir; + } + + set externalLogDir(value: string | undefined) { + this.#externalLogDir = value; + if (value != null) { + Deno.mkdirSync(value, { + recursive: true, + }); + } + } + + write(entry: LogEntry): void { + const { content, level, cause } = entry; + const [message, ...rest] = content; + console[level ?? this.#context.level]( + " ".repeat(this.#indentSize * this.#context.depth) + String(message), + ...(cause != null ? [cause, ...rest] : rest), + ); + } + + push(entry: LogEntry): Disposable { + this.write(entry); + this.#contextStack.push({ + depth: this.#context.depth + 1, + level: entry.level ?? this.#context.level, + }); + return { + [Symbol.dispose]: () => { + this.#contextStack.pop(); + }, + }; + } + + info(message: unknown, ...args: unknown[]) { + this.write({ level: "info", content: [message, ...args] }); + } + + warn(message: unknown, ...args: unknown[]) { + this.write({ level: "warn", content: [message, ...args] }); + } + + error(message: unknown, ...args: unknown[]) { + this.write({ level: "error", content: [message, ...args] }); + } + + builder(message: unknown, ...args: unknown[]): LogEntryBuilder { + const entry: LogEntry = { + content: [message, ...args], + level: "info", + cause: undefined, + }; + const builder: LogEntryBuilder = { + withCause: (cause) => { + entry.cause = cause; + return builder; + }, + withLevel: (level) => { + entry.level = level; + return builder; + }, + setError: (error) => { + if (error) entry.level = "error"; + return builder; + }, + write: () => { + this.write(entry); + }, + }; + return builder; + } + + async createExternalLogStream( + name: string, + options?: { + noTime?: boolean; + }, + ): Promise<ExternalLogStream> { + if (name.includes("/")) { + throw new Error(`External log stream's name (${name}) contains '/'.`); + } + if (this.#externalLogDir == null) { + throw new Error("External log directory is not set."); + } + + const logPath = join( + this.#externalLogDir, + options?.noTime === true + ? name + : `${name}-${toFileNameString(new Date())}`, + ); + + const file = await Deno.open(logPath, { + read: false, + write: true, + append: true, + create: true, + }); + return { + stream: file.writable, + [Symbol.dispose]: file[Symbol.dispose].bind(file), + }; + } + + async createExternalLogStreamsForProgram( + program: string, + ): Promise<{ stdout: WritableStream; stderr: WritableStream } & Disposable> { + const stdout = await this.createExternalLogStream(`${program}-stdout`); + const stderr = await this.createExternalLogStream(`${program}-stderr`); + return { + stdout: stdout.stream, + stderr: stderr.stream, + [Symbol.dispose]: () => { + stdout[Symbol.dispose](); + stderr[Symbol.dispose](); + }, + }; + } +} + diff --git a/deno/base/text.ts b/deno/base/text.ts new file mode 100644 index 0000000..f3e4020 --- /dev/null +++ b/deno/base/text.ts @@ -0,0 +1,3 @@ +export function camelCaseToKebabCase(str: string): string { + return str.replace(/[A-Z]/g, (m) => "-" + m.toLowerCase()); +} diff --git a/deno/deno.json b/deno/deno.json index 558795f..0199e14 100644 --- a/deno/deno.json +++ b/deno/deno.json @@ -1,6 +1,15 @@ { - "workspace": [ "./mail-relay" ], + "workspace": [ "./base", "./mail-relay" ], "tasks": { "compile:mail-relay": "deno task --cwd=mail-relay compile" + }, + "imports": { + "@std/cli": "jsr:@std/cli@^1.0.19", + "@std/csv": "jsr:@std/csv@^1.0.6", + "@std/encoding": "jsr:@std/encoding@^1.0.10", + "@std/expect": "jsr:@std/expect@^1.0.16", + "@std/io": "jsr:@std/io@^0.225.2", + "@std/path": "jsr:@std/path@^1.1.0", + "@std/testing": "jsr:@std/testing@^1.0.13", } } diff --git a/deno/deno.lock b/deno/deno.lock index 020a2c4..336e6e0 100644 --- a/deno/deno.lock +++ b/deno/deno.lock @@ -5,26 +5,32 @@ "jsr:@denosaurs/plug@1": "1.1.0", "jsr:@std/assert@0.217": "0.217.0", "jsr:@std/assert@^1.0.13": "1.0.13", + "jsr:@std/async@^1.0.13": "1.0.13", "jsr:@std/bytes@^1.0.5": "1.0.6", "jsr:@std/cli@^1.0.19": "1.0.19", "jsr:@std/csv@^1.0.6": "1.0.6", + "jsr:@std/data-structures@^1.0.8": "1.0.8", "jsr:@std/encoding@1": "1.0.10", "jsr:@std/encoding@^1.0.10": "1.0.10", "jsr:@std/expect@^1.0.16": "1.0.16", "jsr:@std/fmt@1": "1.0.8", "jsr:@std/fs@1": "1.0.17", + "jsr:@std/fs@^1.0.17": "1.0.17", "jsr:@std/internal@^1.0.6": "1.0.7", "jsr:@std/internal@^1.0.7": "1.0.7", + "jsr:@std/internal@^1.0.8": "1.0.8", "jsr:@std/io@~0.225.2": "0.225.2", "jsr:@std/path@0.217": "0.217.0", "jsr:@std/path@1": "1.1.0", "jsr:@std/path@^1.0.9": "1.1.0", "jsr:@std/path@^1.1.0": "1.1.0", "jsr:@std/streams@^1.0.9": "1.0.9", + "jsr:@std/testing@^1.0.13": "1.0.13", "npm:@aws-sdk/client-s3@^3.821.0": "3.824.0", "npm:@aws-sdk/client-sesv2@^3.821.0": "3.824.0", "npm:@hono/zod-validator@0.7": "0.7.0_hono@4.7.11_zod@3.25.51", "npm:@smithy/fetch-http-handler@^5.0.4": "5.0.4", + "npm:@types/node@*": "22.15.15", "npm:email-addresses@5": "5.0.0", "npm:hono@^4.7.11": "4.7.11", "npm:kysely@~0.28.2": "0.28.2", @@ -43,7 +49,7 @@ "dependencies": [ "jsr:@std/encoding@1", "jsr:@std/fmt", - "jsr:@std/fs", + "jsr:@std/fs@1", "jsr:@std/path@1" ] }, @@ -56,6 +62,9 @@ "jsr:@std/internal@^1.0.6" ] }, + "@std/async@1.0.13": { + "integrity": "1d76ca5d324aef249908f7f7fe0d39aaf53198e5420604a59ab5c035adc97c96" + }, "@std/bytes@1.0.6": { "integrity": "f6ac6adbd8ccd99314045f5703e23af0a68d7f7e58364b47d2c7f408aeb5820a" }, @@ -68,6 +77,9 @@ "jsr:@std/streams" ] }, + "@std/data-structures@1.0.8": { + "integrity": "2fb7219247e044c8fcd51341788547575653c82ae2c759ff209e0263ba7d9b66" + }, "@std/encoding@1.0.10": { "integrity": "8783c6384a2d13abd5e9e87a7ae0520a30e9f56aeeaa3bdf910a3eaaf5c811a1" }, @@ -90,6 +102,9 @@ "@std/internal@1.0.7": { "integrity": "39eeb5265190a7bc5d5591c9ff019490bd1f2c3907c044a11b0d545796158a0f" }, + "@std/internal@1.0.8": { + "integrity": "fc66e846d8d38a47cffd274d80d2ca3f0de71040f855783724bb6b87f60891f5" + }, "@std/io@0.225.2": { "integrity": "3c740cd4ee4c082e6cfc86458f47e2ab7cb353dc6234d5e9b1f91a2de5f4d6c7", "dependencies": [ @@ -110,6 +125,17 @@ "dependencies": [ "jsr:@std/bytes" ] + }, + "@std/testing@1.0.13": { + "integrity": "74418be16f627dfe996937ab0ffbdbda9c1f35534b78724658d981492f121e71", + "dependencies": [ + "jsr:@std/assert@^1.0.13", + "jsr:@std/async", + "jsr:@std/data-structures", + "jsr:@std/fs@^1.0.17", + "jsr:@std/internal@^1.0.8", + "jsr:@std/path@^1.1.0" + ] } }, "npm": { @@ -1127,6 +1153,12 @@ "tslib" ] }, + "@types/node@22.15.15": { + "integrity": "sha512-R5muMcZob3/Jjchn5LcO8jdKwSCbzqmPB6ruBxMcf9kbxtniZHP327s6C37iOfuw8mbKK3cAQa7sEl7afLrQ8A==", + "dependencies": [ + "undici-types" + ] + }, "bowser@2.11.0": { "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==" }, @@ -1152,6 +1184,9 @@ "tslib@2.8.1": { "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" }, + "undici-types@6.21.0": { + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==" + }, "uuid@9.0.1": { "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", "bin": true @@ -1161,17 +1196,19 @@ } }, "workspace": { + "dependencies": [ + "jsr:@std/cli@^1.0.19", + "jsr:@std/csv@^1.0.6", + "jsr:@std/encoding@^1.0.10", + "jsr:@std/expect@^1.0.16", + "jsr:@std/io@~0.225.2", + "jsr:@std/path@^1.1.0", + "jsr:@std/testing@^1.0.13" + ], "members": { "mail-relay": { "dependencies": [ "jsr:@db/sqlite@0.12", - "jsr:@std/cli@^1.0.19", - "jsr:@std/csv@^1.0.6", - "jsr:@std/encoding@^1.0.10", - "jsr:@std/expect@^1.0.16", - "jsr:@std/io@~0.225.2", - "jsr:@std/path@^1.1.0", - "jsr:@std/testing@^1.0.13", "npm:@aws-sdk/client-s3@^3.821.0", "npm:@aws-sdk/client-sesv2@^3.821.0", "npm:@hono/zod-validator@0.7", diff --git a/deno/mail-relay/app.ts b/deno/mail-relay/app.ts index deb72c2..3cac44b 100644 --- a/deno/mail-relay/app.ts +++ b/deno/mail-relay/app.ts @@ -1,81 +1,90 @@ -import { join } from "@std/path"; import { Hono } from "hono"; import { logger as honoLogger } from "hono/logger"; -import log from "./log.ts"; -import config from "./config.ts"; -import { DbService } from "./db.ts"; +import { Logger } from "@crupest/base/log"; + import { AliasRecipientMailHook, FallbackRecipientHook, MailDeliverer, + RecipientFromHeadersHook, } from "./mail.ts"; -import { DovecotMailDeliverer } from "./dovecot/deliver.ts"; -import { CronTask, CronTaskConfig } from "./cron.ts"; +import { DovecotMailDeliverer } from "./dovecot.ts"; import { DumbSmtpServer } from "./dumb-smtp-server.ts"; -export abstract class AppBase { - protected readonly db: DbService; - protected readonly crons: CronTask[] = []; - protected readonly routes: Hono[] = []; - protected readonly inboundDeliverer: MailDeliverer; - protected readonly hono = new Hono(); - - protected abstract readonly outboundDeliverer: MailDeliverer; - - constructor() { - const dataPath = config.get("dataPath"); - Deno.mkdirSync(dataPath, { recursive: true }); - log.path = join(dataPath, "log"); - log.info(config); +export function createInbound( + logger: Logger, + { + fallback, + mailDomain, + aliasFile, + ldaPath, + }: { + fallback: string[]; + mailDomain: string; + aliasFile: string; + ldaPath: string; + }, +) { + const deliverer = new DovecotMailDeliverer(logger, ldaPath); + deliverer.preHooks.push( + new RecipientFromHeadersHook(mailDomain), + new FallbackRecipientHook(new Set(fallback)), + new AliasRecipientMailHook(aliasFile), + ); + return deliverer; +} - this.db = new DbService(join(dataPath, "db.sqlite")); - this.inboundDeliverer = new DovecotMailDeliverer(); - this.inboundDeliverer.preHooks.push( - new FallbackRecipientHook(new Set(config.getList("inboundFallback"))), - new AliasRecipientMailHook(join(dataPath, "aliases.csv")), - ); +export function createHono( + logger: Logger, + outbound: MailDeliverer, + inbound: MailDeliverer, +) { + const hono = new Hono(); - this.hono.onError((err, c) => { - log.error(err); - return c.json({ msg: "Server error, check its log." }, 500); - }); + hono.onError((err, c) => { + logger.error(err); + return c.json({ msg: "Server error, check its log." }, 500); + }); + hono.use(honoLogger()); + hono.post("/send/raw", async (context) => { + const body = await context.req.text(); + if (body.trim().length === 0) { + return context.json({ msg: "Can't send an empty mail." }, 400); + } else { + const result = await outbound.deliverRaw(body); + return context.json({ + awsMessageId: result.awsMessageId, + }); + } + }); + hono.post("/receive/raw", async (context) => { + await inbound.deliverRaw(await context.req.text()); + return context.json({ msg: "Done!" }); + }); - this.hono.use(honoLogger()); - this.hono.post("/send/raw", async (context) => { - const body = await context.req.text(); - if (body.trim().length === 0) { - return context.json({ msg: "Can't send an empty mail." }, 400); - } else { - const result = await this.outboundDeliverer.deliverRaw(body); - return context.json({ - awsMessageId: result.awsMessageId, - }); - } - }); - this.hono.post("/receive/raw", async (context) => { - await this.inboundDeliverer.deliverRaw(await context.req.text()); - return context.json({ "msg": "Done!" }); - }); - } + return hono; +} - createCron(config: CronTaskConfig): CronTask { - const cron = new CronTask(config); - this.crons.push(cron); - return cron; - } +export function createSmtp(logger: Logger, outbound: MailDeliverer) { + return new DumbSmtpServer(logger, outbound); +} - async setup() { - await this.db.migrate() +export async function sendMail(logger: Logger, port: number) { + const decoder = new TextDecoder(); + let text = ""; + for await (const chunk of Deno.stdin.readable) { + text += decoder.decode(chunk); } - serve(): { smtp: DumbSmtpServer; http: Deno.HttpServer } { - const smtp = new DumbSmtpServer(this.outboundDeliverer); - smtp.serve(); - const http = Deno.serve({ - hostname: config.HTTP_HOST, - port: config.HTTP_PORT, - }, this.hono.fetch); - return { smtp, http }; - } + const res = await fetch(`http://127.0.0.1:${port}/send/raw`, { + method: "post", + body: text, + }); + logger.builder(res).setError(!res.ok).write(); + logger + .builder("Body\n" + (await res.text())) + .setError(!res.ok) + .write(); + if (!res.ok) Deno.exit(-1); } diff --git a/deno/mail-relay/aws/app.ts b/deno/mail-relay/aws/app.ts index 1fda64e..685d7a9 100644 --- a/deno/mail-relay/aws/app.ts +++ b/deno/mail-relay/aws/app.ts @@ -1,113 +1,266 @@ +import { join } from "@std/path"; import { parseArgs } from "@std/cli"; import { z } from "zod"; +import { Hono } from "hono"; import { zValidator } from "@hono/zod-validator"; +import { FetchHttpHandler } from "@smithy/fetch-http-handler"; -import log from "../log.ts"; -import config from "../config.ts"; -import { AppBase } from "../app.ts"; -import { AwsContext } from "./context.ts"; +import { Logger } from "@crupest/base/log"; +import { ConfigDefinition, ConfigProvider } from "@crupest/base/config"; +import { CronTask } from "@crupest/base/cron"; + +import { DbService } from "../db.ts"; +import { Mail } from "../mail.ts"; import { - AwsMailDeliverer, AwsMailMessageIdRewriteHook, AwsMailMessageIdSaveHook, -} from "./deliver.ts"; -import { AwsMailRetriever } from "./retriever.ts"; - -export class AwsRelayApp extends AppBase { - readonly #aws = new AwsContext(); - readonly #retriever; - protected readonly outboundDeliverer = new AwsMailDeliverer(this.#aws); - - constructor() { - super(); - this.#retriever = new AwsMailRetriever(this.#aws, this.inboundDeliverer); - - this.outboundDeliverer.preHooks.push( - new AwsMailMessageIdRewriteHook(this.db), - ); - this.outboundDeliverer.postHooks.push( - new AwsMailMessageIdSaveHook(this.db), - ); - - this.hono.post( - `/${config.get("awsInboundPath")}`, - async (ctx, next) => { - const auth = ctx.req.header("Authorization"); - if (auth !== config.get("awsInboundKey")) { - return ctx.json({ "msg": "Bad auth!" }, 403); - } - await next(); - }, - zValidator( - "json", - z.object({ - key: z.string(), - recipients: z.optional(z.array(z.string())), - }), - ), - async (ctx) => { - const { key, recipients } = ctx.req.valid("json"); - await this.#retriever.deliverS3Mail(key, recipients); - return ctx.json({ "msg": "Done!" }); - }, - ); - } +} from "./mail.ts"; +import { AwsMailDeliverer } from "./deliver.ts"; +import { AwsMailFetcher, AwsS3MailConsumer } from "./fetch.ts"; +import { createInbound, createHono, sendMail, createSmtp } from "../app.ts"; - realServe() { - this.createCron({ - name: "live-mail-recycler", - interval: 6 * 3600 * 1000, - callback: () => { - return this.#retriever.recycleLiveMails(); - }, - startNow: true, - }); - - return this.serve(); - } +const PREFIX = "crupest-mail-server"; +const CONFIG_DEFINITIONS = { + dataPath: { + description: "Path to save app persistent data.", + default: ".", + }, + mailDomain: { + description: + "The part after `@` of an address. Used to determine local recipients.", + }, + httpHost: { + description: "Listening address for http server.", + default: "0.0.0.0", + }, + httpPort: { description: "Listening port for http server.", default: "2345" }, + smtpHost: { + description: "Listening address for dumb smtp server.", + default: "127.0.0.1", + }, + smtpPort: { + description: "Listening port for dumb smtp server.", + default: "2346", + }, + ldaPath: { + description: "full path of lda executable", + default: "/dovecot/libexec/dovecot/dovecot-lda", + }, + inboundFallback: { + description: "comma separated addresses used as fallback recipients", + default: "", + }, + awsInboundPath: { + description: "(random set) path for aws sns", + }, + awsInboundKey: { + description: "(random set) http header Authorization for aws sns", + }, + awsRegion: { + description: "aws region", + }, + awsUser: { + description: "aws access key id", + }, + awsPassword: { + description: "aws secret access key", + secret: true, + }, + awsMailBucket: { + description: "aws s3 bucket saving raw mails", + secret: true, + }, +} as const satisfies ConfigDefinition; + +function createAwsOptions({ + user, + password, + region, +}: { + user: string; + password: string; + region: string; +}) { + return { + credentials: () => + Promise.resolve({ + accessKeyId: user, + secretAccessKey: password, + }), + requestHandler: new FetchHttpHandler(), + region, + }; +} - readonly cli = { - "init": (_: unknown) => { - log.info("Just init!"); - return Promise.resolve(); +function createOutbound( + logger: Logger, + awsOptions: ReturnType<typeof createAwsOptions>, + db: DbService, +) { + const deliverer = new AwsMailDeliverer(logger, awsOptions); + deliverer.preHooks.push( + new AwsMailMessageIdRewriteHook(db.messageIdToAws.bind(db)), + ); + deliverer.postHooks.push( + new AwsMailMessageIdSaveHook((original, aws) => + db.addMessageIdMap({ message_id: original, aws_message_id: aws }).then(), + ), + ); + return deliverer; +} + +function setupAwsHono( + hono: Hono, + options: { + path: string; + auth: string; + callback: (s3Key: string, recipients?: string[]) => Promise<void>; + }, +) { + hono.post( + `/${options.path}`, + async (ctx, next) => { + const auth = ctx.req.header("Authorization"); + if (auth !== options.auth) { + return ctx.json({ msg: "Bad auth!" }, 403); + } + await next(); }, - "list-lives": async (_: unknown) => { - const liveMails = await this.#retriever.listLiveMails(); - log.info(`Total ${liveMails.length}:`); - log.info(liveMails.join("\n")); + zValidator( + "json", + z.object({ + key: z.string(), + recipients: z.optional(z.array(z.string())), + }), + ), + async (ctx) => { + const { key, recipients } = ctx.req.valid("json"); + await options.callback(key, recipients); + return ctx.json({ msg: "Done!" }); }, - "recycle-lives": async (_: unknown) => { - await this.#retriever.recycleLiveMails(); + ); +} + +function createCron(fetcher: AwsMailFetcher, consumer: AwsS3MailConsumer) { + return new CronTask({ + name: "live-mail-recycler", + interval: 6 * 3600 * 1000, + callback: () => { + return fetcher.recycleLiveMails(consumer); }, - "serve": async (_: unknown) => { - await this.serve().http.finished; + startNow: true, + }); +} + +function createBaseServices() { + const config = new ConfigProvider(PREFIX, CONFIG_DEFINITIONS); + Deno.mkdirSync(config.get("dataPath"), { recursive: true }); + const logger = new Logger(); + logger.externalLogDir = join(config.get("dataPath"), "log"); + return { config, logger }; +} + +function createAwsFetchOnlyServices() { + const { config, logger } = createBaseServices(); + const awsOptions = createAwsOptions({ + user: config.get("awsUser"), + password: config.get("awsPassword"), + region: config.get("awsRegion"), + }); + const fetcher = new AwsMailFetcher( + logger, + awsOptions, + config.get("awsMailBucket"), + ); + return { config, logger, awsOptions, fetcher }; +} + +function createAwsRecycleOnlyServices() { + const { config, logger, awsOptions, fetcher } = createAwsFetchOnlyServices(); + + const inbound = createInbound(logger, { + fallback: config.getList("inboundFallback"), + ldaPath: config.get("ldaPath"), + aliasFile: join(config.get("dataPath"), "aliases.csv"), + mailDomain: config.get("mailDomain"), + }); + + const recycler = (rawMail: string, _: unknown): Promise<void> => + inbound.deliver({ mail: new Mail(rawMail) }).then(); + + return { config, logger, awsOptions, fetcher, inbound, recycler }; +} +function createAwsServices() { + const { config, logger, inbound, awsOptions, fetcher, recycler } = + createAwsRecycleOnlyServices(); + const dbService = new DbService(join(config.get("dataPath"), "db.sqlite")); + const outbound = createOutbound(logger, awsOptions, dbService); + + return { + config, + logger, + inbound, + dbService, + awsOptions, + fetcher, + recycler, + outbound, + }; +} + +function createServerServices() { + const services = createAwsServices(); + const { logger, config, outbound, inbound, fetcher } = services; + const smtp = createSmtp(logger, outbound); + + const hono = createHono(logger, outbound, inbound); + setupAwsHono(hono, { + path: config.get("awsInboundPath"), + auth: config.get("awsInboundKey"), + callback: (s3Key, recipients) => { + return fetcher.consumeS3Mail(s3Key, (rawMail, _) => + inbound.deliver({ mail: new Mail(rawMail), recipients }).then(), + ); }, - "real-serve": async (_: unknown) => { - await this.realServe().http.finished; + }); + + return { + ...services, + smtp, + hono, + }; +} + +function serve(cron: boolean = false) { + const { config, fetcher, recycler, smtp, hono } = createServerServices(); + smtp.serve({ + hostname: config.get("smtpHost"), + port: config.getInt("smtpPort"), + }); + Deno.serve( + { + hostname: config.get("httpHost"), + port: config.getInt("httpPort"), }, - } as const; + hono.fetch, + ); + + if (cron) { + createCron(fetcher, recycler); + } } -const nonServerCli = { - "sendmail": async (_: unknown) => { - const decoder = new TextDecoder(); - let text = ""; - for await (const chunk of Deno.stdin.readable) { - text += decoder.decode(chunk); - } +async function listLives() { + const { logger, fetcher } = createAwsFetchOnlyServices(); + const liveMails = await fetcher.listLiveMails(); + logger.info(`Total ${liveMails.length}:`); + logger.info(liveMails.join("\n")); +} - const res = await fetch( - `http://localhost:${config.HTTP_PORT}/send/raw`, - { - method: "post", - body: text, - }, - ); - log.infoOrError(!res.ok, res); - log.infoOrError(!res.ok, "Body\n" + await res.text()); - if (!res.ok) Deno.exit(-1); - }, -} as const; +async function recycleLives() { + const { fetcher, recycler } = createAwsRecycleOnlyServices(); + await fetcher.recycleLiveMails(recycler); +} if (import.meta.main) { const args = parseArgs(Deno.args); @@ -116,21 +269,32 @@ if (import.meta.main) { throw new Error("You must specify a command."); } - const command = args._[0]; - - if (command in nonServerCli) { - log.info(`Run non-server command ${command}.`); - await nonServerCli[command as keyof typeof nonServerCli](args); - Deno.exit(0); - } + const command = String(args._[0]); - const app = new AwsRelayApp(); - await app.setup(); - if (command in app.cli) { - log.info(`Run command ${command}.`); - await app.cli[command as keyof AwsRelayApp["cli"]](args); - Deno.exit(0); - } else { - throw new Error(command + " is not a valid command."); + switch (command) { + case "sendmail": { + const { logger, config } = createBaseServices(); + await sendMail(logger, config.getInt("httpPort")); + break; + } + case "list-lives": { + await listLives(); + break; + } + case "recycle-lives": { + await recycleLives(); + break; + } + case "serve": { + serve(); + break; + } + case "real-serve": { + serve(true); + break; + } + default: { + throw new Error(command + " is not a valid command."); + } } } diff --git a/deno/mail-relay/aws/context.ts b/deno/mail-relay/aws/context.ts deleted file mode 100644 index b1e0336..0000000 --- a/deno/mail-relay/aws/context.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { - CopyObjectCommand, - DeleteObjectCommand, - S3Client, -} from "@aws-sdk/client-s3"; -import { FetchHttpHandler } from "@smithy/fetch-http-handler"; - -import config from "../config.ts"; - -export class AwsContext { - readonly credentials = () => - Promise.resolve({ - accessKeyId: config.get("awsUser"), - secretAccessKey: config.get("awsPassword"), - }); - readonly requestHandler = new FetchHttpHandler(); - - get region() { - return config.get("awsRegion"); - } -} - -export async function s3MoveObject( - client: S3Client, - bucket: string, - path: string, - newPath: string, -): Promise<void> { - const copyCommand = new CopyObjectCommand({ - Bucket: bucket, - Key: newPath, - CopySource: `${bucket}/${path}`, - }); - await client.send(copyCommand); - - const deleteCommand = new DeleteObjectCommand({ - Bucket: bucket, - Key: path, - }); - await client.send(deleteCommand); -} diff --git a/deno/mail-relay/aws/deliver.ts b/deno/mail-relay/aws/deliver.ts index 0db5fa8..3e1f162 100644 --- a/deno/mail-relay/aws/deliver.ts +++ b/deno/mail-relay/aws/deliver.ts @@ -1,16 +1,13 @@ -// spellchecker: words sesv2 amazonses +// spellchecker:words sesv2 amazonses -import { SendEmailCommand, SESv2Client } from "@aws-sdk/client-sesv2"; - -import log from "../log.ts"; -import { DbService } from "../db.ts"; import { - Mail, - MailDeliverContext, - MailDeliverHook, - SyncMailDeliverer, -} from "../mail.ts"; -import { AwsContext } from "./context.ts"; + SendEmailCommand, + SESv2Client, + SESv2ClientConfig, +} from "@aws-sdk/client-sesv2"; + +import { Logger } from "@crupest/base/log"; +import { Mail, MailDeliverContext, SyncMailDeliverer } from "../mail.ts"; declare module "../mail.ts" { interface MailDeliverResult { @@ -18,61 +15,15 @@ declare module "../mail.ts" { } } -export class AwsMailMessageIdRewriteHook implements MailDeliverHook { - readonly #db; - - constructor(db: DbService) { - this.#db = db; - } - - async callback(context: MailDeliverContext): Promise<void> { - log.info("Rewrite message ids..."); - const addresses = context.mail.simpleFindAllAddresses(); - log.info(`Addresses found in mail: ${addresses.join(", ")}.`); - for (const address of addresses) { - const awsMessageId = await this.#db.messageIdToAws(address); - if (awsMessageId != null && awsMessageId.length !== 0) { - log.info(`Rewrite ${address} to ${awsMessageId}.`); - context.mail.raw = context.mail.raw.replaceAll(address, awsMessageId); - } - } - log.info("Done rewrite message ids."); - } -} - -export class AwsMailMessageIdSaveHook implements MailDeliverHook { - readonly #db; - - constructor(db: DbService) { - this.#db = db; - } - - async callback(context: MailDeliverContext): Promise<void> { - log.info("Save aws message ids..."); - const messageId = context.mail.startSimpleParse().sections().headers() - .messageId(); - if (messageId == null) { - log.info("Original mail does not have message id. Skip saving."); - return; - } - if (context.result.awsMessageId != null) { - log.info(`Saving ${messageId} => ${context.result.awsMessageId}.`); - await this.#db.addMessageIdMap({ - message_id: messageId, - aws_message_id: context.result.awsMessageId, - }); - } - log.info("Done save message ids."); - } -} - export class AwsMailDeliverer extends SyncMailDeliverer { readonly name = "aws"; + readonly #logger; readonly #aws; readonly #ses; - constructor(aws: AwsContext) { - super(); + constructor(logger: Logger, aws: SESv2ClientConfig) { + super(logger); + this.#logger = logger; this.#aws = aws; this.#ses = new SESv2Client(aws); } @@ -81,7 +32,7 @@ export class AwsMailDeliverer extends SyncMailDeliverer { mail: Mail, context: MailDeliverContext, ): Promise<void> { - log.info("Begin to call aws send-email api..."); + this.#logger.info("Begin to call aws send-email api..."); try { const sendCommand = new SendEmailCommand({ @@ -92,7 +43,7 @@ export class AwsMailDeliverer extends SyncMailDeliverer { const res = await this.#ses.send(sendCommand); if (res.MessageId == null) { - log.warn("Aws send-email returns no message id."); + this.#logger.warn("Aws send-email returns no message id."); } else { context.result.awsMessageId = `${res.MessageId}@${this.#aws.region}.amazonses.com`; diff --git a/deno/mail-relay/aws/fetch.ts b/deno/mail-relay/aws/fetch.ts new file mode 100644 index 0000000..ef1ba5f --- /dev/null +++ b/deno/mail-relay/aws/fetch.ts @@ -0,0 +1,131 @@ +import { + CopyObjectCommand, + DeleteObjectCommand, + GetObjectCommand, + ListObjectsV2Command, + S3Client, + S3ClientConfig, +} from "@aws-sdk/client-s3"; + +import { toFileNameString } from "@crupest/base/date"; +import { Logger } from "@crupest/base/log"; + +import { Mail } from "../mail.ts"; + +async function s3MoveObject( + client: S3Client, + bucket: string, + path: string, + newPath: string, +): Promise<void> { + const copyCommand = new CopyObjectCommand({ + Bucket: bucket, + Key: newPath, + CopySource: `${bucket}/${path}`, + }); + await client.send(copyCommand); + + const deleteCommand = new DeleteObjectCommand({ + Bucket: bucket, + Key: path, + }); + await client.send(deleteCommand); +} + +const AWS_SES_S3_SETUP_TAG = "AMAZON_SES_SETUP_NOTIFICATION"; + +export type AwsS3MailConsumer = ( + rawMail: string, + s3Key: string, +) => Promise<void>; + +export class AwsMailFetcher { + readonly #livePrefix = "mail/live/"; + readonly #archivePrefix = "mail/archive/"; + readonly #logger; + readonly #s3; + readonly #bucket; + + constructor(logger: Logger, aws: S3ClientConfig, bucket: string) { + this.#logger = logger; + this.#s3 = new S3Client(aws); + this.#bucket = bucket; + } + + async listLiveMails(): Promise<string[]> { + this.#logger.info("Begin to retrieve live mails."); + + const listCommand = new ListObjectsV2Command({ + Bucket: this.#bucket, + Prefix: this.#livePrefix, + }); + const res = await this.#s3.send(listCommand); + + if (res.Contents == null) { + this.#logger.warn("Listing live mails in S3 returns null Content."); + return []; + } + + const result: string[] = []; + for (const object of res.Contents) { + if (object.Key == null) { + this.#logger.warn( + "Listing live mails in S3 returns an object with no Key.", + ); + continue; + } + + if (object.Key.endsWith(AWS_SES_S3_SETUP_TAG)) continue; + + result.push(object.Key.slice(this.#livePrefix.length)); + } + return result; + } + + async consumeS3Mail(s3Key: string, consumer: AwsS3MailConsumer) { + this.#logger.info(`Begin to consume s3 mail ${s3Key} ...`); + + this.#logger.info(`Fetching s3 mail ${s3Key}...`); + const mailPath = `${this.#livePrefix}${s3Key}`; + const command = new GetObjectCommand({ + Bucket: this.#bucket, + Key: mailPath, + }); + const res = await this.#s3.send(command); + + if (res.Body == null) { + throw new Error("S3 mail returns a null body."); + } + + const rawMail = await res.Body.transformToString(); + this.#logger.info(`Done fetching s3 mail ${s3Key}.`); + + this.#logger.info(`Calling consumer...`); + await consumer(rawMail, s3Key); + this.#logger.info(`Done consuming s3 mail ${s3Key}.`); + + const date = new Mail(rawMail) + .startSimpleParse(this.#logger) + .sections() + .headers() + .date(); + const dateString = + date != null ? toFileNameString(date, true) : "invalid-date"; + const newPath = `${this.#archivePrefix}${dateString}/${s3Key}`; + + this.#logger.info(`Archiving s3 mail ${s3Key} to ${newPath}...`); + await s3MoveObject(this.#s3, this.#bucket, mailPath, newPath); + this.#logger.info(`Done archiving s3 mail ${s3Key}.`); + + this.#logger.info(`Done consuming s3 mail ${s3Key}.`); + } + + async recycleLiveMails(consumer: AwsS3MailConsumer) { + this.#logger.info("Begin to recycle live mails..."); + const mails = await this.listLiveMails(); + this.#logger.info(`Found ${mails.length} live mails`); + for (const s3Key of mails) { + await this.consumeS3Mail(s3Key, consumer); + } + } +} diff --git a/deno/mail-relay/aws/mail.ts b/deno/mail-relay/aws/mail.ts new file mode 100644 index 0000000..d2cfad1 --- /dev/null +++ b/deno/mail-relay/aws/mail.ts @@ -0,0 +1,53 @@ +import { MailDeliverContext, MailDeliverHook } from "../mail.ts"; + +export class AwsMailMessageIdRewriteHook implements MailDeliverHook { + readonly #lookup; + + constructor(lookup: (origin: string) => Promise<string | null>) { + this.#lookup = lookup; + } + + async callback(context: MailDeliverContext): Promise<void> { + context.logger.info("Rewrite message ids..."); + const addresses = context.mail.simpleFindAllAddresses(); + context.logger.info(`Addresses found in mail: ${addresses.join(", ")}.`); + for (const address of addresses) { + const awsMessageId = await this.#lookup(address); + if (awsMessageId != null && awsMessageId.length !== 0) { + context.logger.info(`Rewrite ${address} to ${awsMessageId}.`); + context.mail.raw = context.mail.raw.replaceAll(address, awsMessageId); + } + } + context.logger.info("Done rewrite message ids."); + } +} + +export class AwsMailMessageIdSaveHook implements MailDeliverHook { + readonly #record; + + constructor(record: (original: string, aws: string) => Promise<void>) { + this.#record = record; + } + + async callback(context: MailDeliverContext): Promise<void> { + context.logger.info("Save aws message ids..."); + const messageId = context.mail + .startSimpleParse(context.logger) + .sections() + .headers() + .messageId(); + if (messageId == null) { + context.logger.info( + "Original mail does not have message id. Skip saving.", + ); + return; + } + if (context.result.awsMessageId != null) { + context.logger.info( + `Saving ${messageId} => ${context.result.awsMessageId}.`, + ); + await this.#record(messageId, context.result.awsMessageId); + } + context.logger.info("Done save message ids."); + } +} diff --git a/deno/mail-relay/aws/retriever.ts b/deno/mail-relay/aws/retriever.ts deleted file mode 100644 index 756cfc3..0000000 --- a/deno/mail-relay/aws/retriever.ts +++ /dev/null @@ -1,100 +0,0 @@ -/// <reference types="npm:@types/node" /> - -import { - GetObjectCommand, - ListObjectsV2Command, - S3Client, -} from "@aws-sdk/client-s3"; - -import log from "../log.ts"; -import config from "../config.ts"; -import "../better-js.ts"; - -import { Mail, MailDeliverer } from "../mail.ts"; -import { AwsContext, s3MoveObject } from "./context.ts"; - -const AWS_SES_S3_SETUP_TAG = "AMAZON_SES_SETUP_NOTIFICATION"; - -export class AwsMailRetriever { - readonly liveMailPrefix = "mail/live/"; - readonly archiveMailPrefix = "mail/archive/"; - readonly mailBucket = config.get("awsMailBucket"); - - readonly #s3; - - constructor( - aws: AwsContext, - public readonly inboundDeliverer: MailDeliverer, - ) { - this.#s3 = new S3Client(aws); - } - - async listLiveMails(): Promise<string[]> { - log.info("Begin to retrieve live mails."); - - const listCommand = new ListObjectsV2Command({ - Bucket: this.mailBucket, - Prefix: this.liveMailPrefix, - }); - const res = await this.#s3.send(listCommand); - - if (res.Contents == null) { - log.warn("Listing live mails in S3 returns null Content."); - return []; - } - - const result: string[] = []; - for (const object of res.Contents) { - if (object.Key == null) { - log.warn("Listing live mails in S3 returns an object with no Key."); - continue; - } - - if (object.Key.endsWith(AWS_SES_S3_SETUP_TAG)) continue; - - result.push(object.Key.slice(this.liveMailPrefix.length)); - } - return result; - } - - async deliverS3Mail(s3Key: string, recipients: string[] = []) { - log.info(`Begin to deliver s3 mail ${s3Key} to ${recipients.join(" ")}...`); - - log.info(`Fetching s3 mail ${s3Key}...`); - const mailPath = `${this.liveMailPrefix}${s3Key}`; - const command = new GetObjectCommand({ - Bucket: this.mailBucket, - Key: mailPath, - }); - const res = await this.#s3.send(command); - - if (res.Body == null) { - throw new Error("S3 mail returns a null body."); - } - - const rawMail = await res.Body.transformToString(); - log.info(`Done fetching s3 mail ${s3Key}.`); - - log.info(`Delivering s3 mail ${s3Key}...`); - const mail = new Mail(rawMail); - await this.inboundDeliverer.deliver({ mail, recipients: recipients }); - log.info(`Done delivering s3 mail ${s3Key}.`); - - const date = mail.startSimpleParse().sections().headers().date(); - const dateString = date?.toFileNameString(true) ?? "invalid-date"; - const newPath = `${this.archiveMailPrefix}${dateString}/${s3Key}`; - - log.info(`Archiving s3 mail ${s3Key} to ${newPath}...`); - await s3MoveObject(this.#s3, this.mailBucket, mailPath, newPath); - log.info(`Done delivering s3 mail ${s3Key}...`); - } - - async recycleLiveMails() { - log.info("Begin to recycle live mails..."); - const mails = await this.listLiveMails(); - log.info(`Found ${mails.length} live mails`); - for (const s3Key of mails) { - await this.deliverS3Mail(s3Key); - } - } -} diff --git a/deno/mail-relay/better-js.ts b/deno/mail-relay/better-js.ts deleted file mode 100644 index c424a6e..0000000 --- a/deno/mail-relay/better-js.ts +++ /dev/null @@ -1,14 +0,0 @@ -declare global { - interface Date { - toFileNameString(dateOnly?: boolean): string; - } -} - -Object.defineProperty(Date.prototype, "toFileNameString", { - value: function (this: Date, dateOnly?: boolean) { - const str = this.toISOString(); - return dateOnly === true - ? str.slice(0, str.indexOf("T")) - : str.replaceAll(/:|\./g, "-"); - }, -}); diff --git a/deno/mail-relay/config.ts b/deno/mail-relay/config.ts deleted file mode 100644 index d58b163..0000000 --- a/deno/mail-relay/config.ts +++ /dev/null @@ -1,103 +0,0 @@ -export const APP_PREFIX = "crupest"; -export const APP_NAME = "mail-server"; - -export interface ConfigItemDefinition { - description: string; - default?: string; - secret?: boolean; -} - -export const CONFIG_DEFINITIONS = { - mailDomain: { - description: "the part after `@` of an address", - }, - dataPath: { - description: "path to save app persistent data", - }, - ldaPath: { - description: "full path of lda executable", - "default": "/dovecot/libexec/dovecot/dovecot-lda", - }, - inboundFallback: { - description: "comma separated addresses used as fallback recipients", - "default": "", - }, - awsInboundPath: { - description: "(random set) path for aws sns", - }, - awsInboundKey: { - description: "(random set) http header Authorization for aws sns", - }, - awsRegion: { - description: "aws region", - }, - awsUser: { - description: "aws access key id", - }, - awsPassword: { - description: "aws secret access key", - secret: true, - }, - awsMailBucket: { - description: "aws s3 bucket saving raw mails", - secret: true, - }, -} as const satisfies Record<string, ConfigItemDefinition>; - -type ConfigDefinitions = typeof CONFIG_DEFINITIONS; -type ConfigNames = keyof ConfigDefinitions; -type ConfigMap = { - [K in ConfigNames]: ConfigDefinitions[K] & { - readonly env: string; - readonly value: string; - }; -}; - -function resolveConfig(): ConfigMap { - const result: Record<string, ConfigMap[ConfigNames]> = {}; - for (const [name, def] of Object.entries(CONFIG_DEFINITIONS)) { - const env = `${APP_PREFIX}-${APP_NAME}-${ - name.replace(/[A-Z]/g, (m) => "-" + m.toLowerCase()) - }`.replaceAll("-", "_").toUpperCase(); - const value = Deno.env.get(env) ?? (def as ConfigItemDefinition).default; - if (value == null) { - throw new Error(`Required env ${env} (${def.description}) is not set.`); - } - result[name] = { ...def, env, value }; - } - return result as ConfigMap; -} - -export class Config { - #config = resolveConfig(); - - readonly HTTP_HOST = "0.0.0.0"; - readonly HTTP_PORT = 2345; - readonly SMTP_HOST = "127.0.0.1"; - readonly SMTP_PORT = 2346; - - getAllConfig<K extends ConfigNames>(key: K): ConfigMap[K] { - return this.#config[key]; - } - - get(key: ConfigNames): string { - return this.getAllConfig(key).value; - } - - getList(key: ConfigNames, separator: string = ","): string[] { - const value = this.get(key); - if (value.length === 0) return []; - return value.split(separator); - } - - [Symbol.for("Deno.customInspect")]() { - return Object.entries(this.#config).map(([key, item]) => - `${key} [env: ${item.env}]: ${ - (item as ConfigItemDefinition).secret === true ? "***" : item.value - }` - ).join("\n"); - } -} - -const config = new Config(); -export default config; diff --git a/deno/mail-relay/deno.json b/deno/mail-relay/deno.json index 9066b33..e03ba93 100644 --- a/deno/mail-relay/deno.json +++ b/deno/mail-relay/deno.json @@ -1,4 +1,5 @@ { + "version": "0.1.0", "tasks": { "run": "deno run -A aws/app.ts", "test": "deno test -A", @@ -10,13 +11,6 @@ "@db/sqlite": "jsr:@db/sqlite@^0.12.0", "@hono/zod-validator": "npm:@hono/zod-validator@^0.7.0", "@smithy/fetch-http-handler": "npm:@smithy/fetch-http-handler@^5.0.4", - "@std/cli": "jsr:@std/cli@^1.0.19", - "@std/csv": "jsr:@std/csv@^1.0.6", - "@std/encoding": "jsr:@std/encoding@^1.0.10", - "@std/expect": "jsr:@std/expect@^1.0.16", - "@std/io": "jsr:@std/io@^0.225.2", - "@std/path": "jsr:@std/path@^1.1.0", - "@std/testing": "jsr:@std/testing@^1.0.13", "email-addresses": "npm:email-addresses@^5.0.0", "hono": "npm:hono@^4.7.11", "kysely": "npm:kysely@^0.28.2", diff --git a/deno/mail-relay/dovecot/deliver.ts b/deno/mail-relay/dovecot.ts index 92bdc58..cb63766 100644 --- a/deno/mail-relay/dovecot/deliver.ts +++ b/deno/mail-relay/dovecot.ts @@ -1,29 +1,27 @@ import { basename } from "@std/path"; -import config from "../config.ts"; -import log from "../log.ts"; +import { Logger } from "@crupest/base/log"; + import { Mail, MailDeliverContext, MailDeliverer, - RecipientFromHeadersHook, -} from "../mail.ts"; +} from "./mail.ts"; export class DovecotMailDeliverer extends MailDeliverer { readonly name = "dovecot"; + readonly #ldaPath; - constructor() { - super(); - this.preHooks.push( - new RecipientFromHeadersHook(), - ); + constructor(logger: Logger, ldaPath: string) { + super(logger); + this.#ldaPath = ldaPath; } protected override async doDeliver( mail: Mail, context: MailDeliverContext, ): Promise<void> { - const ldaPath = config.get("ldaPath"); + const ldaPath = this.#ldaPath; const ldaBinName = basename(ldaPath); const utf8Stream = mail.toUtf8Bytes(); @@ -35,12 +33,12 @@ export class DovecotMailDeliverer extends MailDeliverer { return; } - log.info(`Deliver to dovecot users: ${recipients.join(", ")}.`); + this.logger.info(`Deliver to dovecot users: ${recipients.join(", ")}.`); for (const recipient of recipients) { try { const commandArgs = ["-d", recipient]; - log.info( + this.logger.info( `Run ${ldaBinName} ${commandArgs.join(" ")}...`, ); @@ -52,9 +50,11 @@ export class DovecotMailDeliverer extends MailDeliverer { }); const ldaProcess = ldaCommand.spawn(); - using logFiles = await log.openLogForProgram(ldaBinName); - ldaProcess.stdout.pipeTo(logFiles.stdout.writable); - ldaProcess.stderr.pipeTo(logFiles.stderr.writable); + using logFiles = await this.logger.createExternalLogStreamsForProgram( + ldaBinName, + ); + ldaProcess.stdout.pipeTo(logFiles.stdout); + ldaProcess.stderr.pipeTo(logFiles.stderr); const stdinWriter = ldaProcess.stdin.getWriter(); await stdinWriter.write(utf8Stream); @@ -97,6 +97,6 @@ export class DovecotMailDeliverer extends MailDeliverer { } } - log.info("Done handling all recipients."); + this.logger.info("Done handling all recipients."); } } diff --git a/deno/mail-relay/dumb-smtp-server.ts b/deno/mail-relay/dumb-smtp-server.ts index 6c63f5c..66c2f7c 100644 --- a/deno/mail-relay/dumb-smtp-server.ts +++ b/deno/mail-relay/dumb-smtp-server.ts @@ -1,32 +1,39 @@ -import config from "./config.ts"; -import log from "./log.ts"; +import { Logger } from "@crupest/base/log"; import { MailDeliverer } from "./mail.ts"; const CRLF = "\r\n"; -const SERVER_NAME = `[${config.SMTP_HOST}]:${config.SMTP_PORT}`; - -const RESPONSES = { - "READY": `220 ${SERVER_NAME} SMTP Ready`, - "EHLO": `250 ${SERVER_NAME}`, - "MAIL": "250 2.1.0 Sender OK", - "RCPT": "250 2.1.5 Recipient OK", - "DATA": "354 Start mail input; end with <CRLF>.<CRLF>", - "QUIT": `211 2.0.0 ${SERVER_NAME} closing connection`, - "INVALID": "500 5.5.1 Error: command not recognized", -} as const; +function createResponses(host: string, port: number | string) { + const serverName = `[${host}]:${port}`; + return { + serverName, + READY: `220 ${serverName} SMTP Ready`, + EHLO: `250 ${serverName}`, + MAIL: "250 2.1.0 Sender OK", + RCPT: "250 2.1.5 Recipient OK", + DATA: "354 Start mail input; end with <CRLF>.<CRLF>", + QUIT: `211 2.0.0 ${serverName} closing connection`, + INVALID: "500 5.5.1 Error: command not recognized", + } as const; +} export class DumbSmtpServer { - #deliverer: MailDeliverer; - - constructor(deliverer: MailDeliverer) { + #logger; + #deliverer; + #responses: ReturnType<typeof createResponses> = createResponses( + "invalid", + "invalid", + ); + + constructor(logger: Logger, deliverer: MailDeliverer) { + this.#logger = logger; this.#deliverer = deliverer; } async #handleConnection(conn: Deno.Conn) { using disposeStack = new DisposableStack(); disposeStack.defer(() => { - log.info("Close smtp session tcp connection."); + this.#logger.info("Close smtp session tcp connection."); conn.close(); }); const writer = conn.writable.getWriter(); @@ -42,7 +49,7 @@ export class DumbSmtpServer { let buffer: string = ""; let rawMail: string | null = null; - await send(RESPONSES["READY"]); + await send(this.#responses["READY"]); while (true) { const { value, done } = await reader.read(); @@ -58,36 +65,36 @@ export class DumbSmtpServer { buffer = buffer.slice(eolPos + CRLF.length); if (rawMail == null) { - log.info("Smtp server received line:", line); + this.#logger.info("Smtp server received line:", line); const upperLine = line.toUpperCase(); if (upperLine.startsWith("EHLO") || upperLine.startsWith("HELO")) { - await send(RESPONSES["EHLO"]); + await send(this.#responses["EHLO"]); } else if (upperLine.startsWith("MAIL FROM:")) { - await send(RESPONSES["MAIL"]); + await send(this.#responses["MAIL"]); } else if (upperLine.startsWith("RCPT TO:")) { - await send(RESPONSES["RCPT"]); + await send(this.#responses["RCPT"]); } else if (upperLine === "DATA") { - await send(RESPONSES["DATA"]); - log.info("Begin to receive mail data..."); + await send(this.#responses["DATA"]); + this.#logger.info("Begin to receive mail data..."); rawMail = ""; } else if (upperLine === "QUIT") { - await send(RESPONSES["QUIT"]); + await send(this.#responses["QUIT"]); return; } else { - log.warn("Smtp server command unrecognized:", line); - await send(RESPONSES["INVALID"]); + this.#logger.warn("Smtp server command unrecognized:", line); + await send(this.#responses["INVALID"]); return; } } else { if (line === ".") { try { - log.info("Done receiving mail data, begin to relay..."); + this.#logger.info("Done receiving mail data, begin to relay..."); const { message } = await this.#deliverer.deliverRaw(rawMail); await send(`250 2.6.0 ${message}`); rawMail = null; - log.info("Done SMTP mail session."); + this.#logger.info("Done SMTP mail session."); } catch (err) { - log.info(err); + this.#logger.info(err); await send("554 5.3.0 Error: check server log"); return; } @@ -100,19 +107,21 @@ export class DumbSmtpServer { } } - async serve() { - const listener = Deno.listen({ - hostname: config.SMTP_HOST, - port: config.SMTP_PORT, - }); - listener.unref(); - log.info(`Dumb SMTP server starts running on port ${config.SMTP_PORT}.`); + async serve(options: { + hostname: string, + port: number + }) { + const listener = Deno.listen(options); + this.#responses = createResponses(options.hostname, options.port); + this.#logger.info( + `Dumb SMTP server starts running on ${this.#responses.serverName}.`, + ); for await (const conn of listener) { try { await this.#handleConnection(conn); } catch (cause) { - log.error("One smtp connection session throws an error " + cause); + this.#logger.error("One smtp connection session throws an error " + cause); } } } diff --git a/deno/mail-relay/log.ts b/deno/mail-relay/log.ts deleted file mode 100644 index ce27eca..0000000 --- a/deno/mail-relay/log.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { join } from "@std/path"; -import { toWritableStream, Writer } from "@std/io"; - -import "./better-js.ts"; - -export interface LogOptions { - time?: Date; - error?: boolean; -} - -export type LogFile = Pick<Deno.FsFile, "writable"> & Disposable; - -export class Log { - #path: string | null = null; - - #wrapWriter(writer: Writer): LogFile { - return { - writable: toWritableStream(writer, { autoClose: false }), - [Symbol.dispose]() {}, - }; - } - - #stdoutWrapper: LogFile = this.#wrapWriter(Deno.stdout); - #stderrWrapper: LogFile = this.#wrapWriter(Deno.stderr); - - constructor() { - } - - get path() { - return this.#path; - } - - set path(path) { - this.#path = path; - if (path != null) { - Deno.mkdirSync(path, { recursive: true }); - } - } - - infoOrError(isError: boolean, ...args: unknown[]) { - this[isError ? "error" : "info"].call(this, ...args); - } - - info(...args: unknown[]) { - console.log(...args); - } - - warn(...args: unknown[]) { - console.warn(...args); - } - - error(...args: unknown[]) { - console.error(...args); - } - - #extractOptions(options?: LogOptions): Required<LogOptions> { - return { - time: options?.time ?? new Date(), - error: options?.error ?? false, - }; - } - - async openLog( - prefix: string, - suffix: string, - options?: LogOptions, - ): Promise<LogFile> { - if (prefix.includes("/")) { - throw new Error(`Log file prefix ${prefix} contains '/'.`); - } - if (suffix.includes("/")) { - throw new Error(`Log file suffix ${suffix} contains '/'.`); - } - - const { time, error } = this.#extractOptions(options); - if (this.#path == null) { - return error ? this.#stderrWrapper : this.#stdoutWrapper; - } - - const logPath = join( - this.#path, - `${prefix}-${time.toFileNameString()}-${suffix}`, - ); - return await Deno.open(logPath, { - read: false, - write: true, - append: true, - create: true, - }); - } - - async openLogForProgram( - program: string, - options?: Omit<LogOptions, "error">, - ): Promise<{ stdout: LogFile; stderr: LogFile } & Disposable> { - const stdout = await this.openLog(program, "stdout", { - ...options, - error: false, - }); - const stderr = await this.openLog(program, "stderr", { - ...options, - error: true, - }); - return { - stdout, - stderr, - [Symbol.dispose]: () => { - stdout[Symbol.dispose](); - stderr[Symbol.dispose](); - }, - }; - } -} - -const log = new Log(); -export default log; diff --git a/deno/mail-relay/mail.test.ts b/deno/mail-relay/mail.test.ts index ee275af..6f3cd13 100644 --- a/deno/mail-relay/mail.test.ts +++ b/deno/mail-relay/mail.test.ts @@ -1,6 +1,8 @@ import { describe, it } from "@std/testing/bdd"; import { expect, fn } from "@std/expect"; +import { Logger } from "@crupest/base/log"; + import { Mail, MailDeliverContext, MailDeliverer } from "./mail.ts"; const mockDate = "Fri, 02 May 2025 08:33:02 +0000"; @@ -71,27 +73,12 @@ describe("Mail", () => { it("simple parse headers", () => { expect( - new Mail(mockMailStr).startSimpleParse().sections().headers(), + new Mail(mockMailStr).startSimpleParse().sections().headers().fields, ).toEqual(mockHeaders.map( (h) => [h[0], " " + h[1].replaceAll("\n", "")], )); }); - it("append headers", () => { - const mail = new Mail(mockMailStr); - const mockMoreHeaders = [["abc", "123"], ["def", "456"]] satisfies [ - string, - string, - ][]; - mail.appendHeaders(mockMoreHeaders); - - expect(mail.raw).toBe( - mockHeaderStr + "\n" + - mockMoreHeaders.map((h) => h[0] + ": " + h[1]).join("\n") + - "\n\n" + mockBodyStr, - ); - }); - it("parse recipients", () => { const mail = new Mail(mockMailStr); expect([...mail.startSimpleParse().sections().headers().recipients()]) @@ -134,7 +121,7 @@ describe("MailDeliverer", () => { return Promise.resolve(); }) as MailDeliverer["doDeliver"]; } - const mockDeliverer = new MockMailDeliverer(); + const mockDeliverer = new MockMailDeliverer(new Logger()); it("deliver success", async () => { await mockDeliverer.deliverRaw(mockMailStr); diff --git a/deno/mail-relay/mail.ts b/deno/mail-relay/mail.ts index af0df40..8c2e067 100644 --- a/deno/mail-relay/mail.ts +++ b/deno/mail-relay/mail.ts @@ -2,24 +2,19 @@ import { encodeBase64 } from "@std/encoding/base64"; import { parse } from "@std/csv/parse"; import emailAddresses from "email-addresses"; -import log from "./log.ts"; -import config from "./config.ts"; +import { Logger } from "@crupest/base/log"; -class MailSimpleParseError extends Error { - constructor( - message: string, - public readonly text: string, - public readonly lineNumber?: number, - options?: ErrorOptions, - ) { - if (lineNumber != null) message += `(at line ${lineNumber})`; - super(message, options); +class MailSimpleParseError extends Error { } + +class MailSimpleParsedHeaders { + #logger + + constructor(logger: Logger | undefined, public fields: [key:string, value: string][]) { + this.#logger = logger; } -} -class MailSimpleParsedHeaders extends Array<[key: string, value: string]> { getFirst(fieldKey: string): string | undefined { - for (const [key, value] of this) { + for (const [key, value] of this.fields) { if (key.toLowerCase() === fieldKey.toLowerCase()) return value; } return undefined; @@ -33,7 +28,7 @@ class MailSimpleParsedHeaders extends Array<[key: string, value: string]> { if (match != null) { return match[1]; } else { - console.warn("Invalid message-id header of mail: ", messageIdField); + this.#logger?.warn("Invalid message-id header of mail: ", messageIdField); return undefined; } } @@ -44,7 +39,7 @@ class MailSimpleParsedHeaders extends Array<[key: string, value: string]> { const date = new Date(dateField); if (invalidToUndefined && isNaN(date.getTime())) { - log.warn(`Invalid date string (${dateField}) found in header.`); + this.#logger?.warn(`Invalid date string (${dateField}) found in header.`); return undefined; } return date; @@ -54,15 +49,16 @@ class MailSimpleParsedHeaders extends Array<[key: string, value: string]> { const domain = options?.domain; const headers = options?.headers ?? ["to", "cc", "bcc", "x-original-to"]; const recipients = new Set<string>(); - for (const [key, value] of this) { + for (const [key, value] of this.fields) { if (headers.includes(key.toLowerCase())) { - emailAddresses.parseAddressList(value)?.flatMap((a) => - a.type === "mailbox" ? a : a.addresses - )?.forEach(({ address }) => { - if (domain == null || address.endsWith(domain)) { - recipients.add(address); - } - }); + emailAddresses + .parseAddressList(value) + ?.flatMap((a) => (a.type === "mailbox" ? a : a.addresses)) + ?.forEach(({ address }) => { + if (domain == null || address.endsWith(domain)) { + recipients.add(address); + } + }); } } return recipients; @@ -75,19 +71,22 @@ class MailSimpleParsedSections { eol: string; sep: string; - constructor(raw: string) { + #logger + + constructor(logger: Logger | undefined, raw: string) { + this.#logger = logger + const twoEolMatch = raw.match(/(\r?\n)(\r?\n)/); if (twoEolMatch == null) { throw new MailSimpleParseError( "No header/body section separator (2 successive EOLs) found.", - raw, ); } const [eol, sep] = [twoEolMatch[1], twoEolMatch[2]]; if (eol !== sep) { - log.warn("Different EOLs (\\r\\n, \\n) found."); + logger?.warn("Different EOLs (\\r\\n, \\n) found."); } this.header = raw.slice(0, twoEolMatch.index!); @@ -97,7 +96,7 @@ class MailSimpleParsedSections { } headers(): MailSimpleParsedHeaders { - const headers = new MailSimpleParsedHeaders(); + const headers = [] as [key:string, value: string][]; let field: string | null = null; let lineNumber = 1; @@ -107,9 +106,7 @@ class MailSimpleParsedSections { const sepPos = field.indexOf(":"); if (sepPos === -1) { throw new MailSimpleParseError( - "No ':' in the header field.", - this.header, - lineNumber, + `No ':' in the header line: ${field}`, ); } headers.push([field.slice(0, sepPos).trim(), field.slice(sepPos + 1)]); @@ -119,11 +116,7 @@ class MailSimpleParsedSections { for (const line of this.header.trimEnd().split(/\r?\n|\r/)) { if (line.match(/^\s/)) { if (field == null) { - throw new MailSimpleParseError( - "Header field starts with a space.", - this.header, - lineNumber, - ); + throw new MailSimpleParseError("Header section starts with a space."); } field += line; } else { @@ -135,7 +128,7 @@ class MailSimpleParsedSections { handleField(); - return headers; + return new MailSimpleParsedHeaders(this.#logger, headers); } } @@ -151,23 +144,14 @@ export class Mail { return encodeBase64(this.raw); } - startSimpleParse() { - return { sections: () => new MailSimpleParsedSections(this.raw) }; + startSimpleParse(logger?: Logger) { + return { sections: () => new MailSimpleParsedSections(logger, this.raw) }; } simpleFindAllAddresses(): string[] { const re = /,?\<?([a-z0-9_'+\-\.]+\@[a-z0-9_'+\-\.]+)\>?,?/ig return [...this.raw.matchAll(re)].map(m => m[1]) } - - // TODO: Add folding. - appendHeaders(headers: [key: string, value: string][]) { - const { header, body, sep, eol } = this.startSimpleParse().sections(); - - this.raw = header + eol + - headers.map(([k, v]) => `${k}: ${v}`).join(eol) + eol + sep + - body; - } } export type MailDeliverResultKind = "done" | "fail"; @@ -203,7 +187,10 @@ export class MailDeliverContext { readonly recipients: Set<string> = new Set(); readonly result; - constructor(public mail: Mail) { + constructor( + public readonly logger: Logger, + public mail: Mail, + ) { this.result = new MailDeliverResult(this.mail); } } @@ -217,6 +204,8 @@ export abstract class MailDeliverer { preHooks: MailDeliverHook[] = []; postHooks: MailDeliverHook[] = []; + constructor(protected readonly logger: Logger) { } + protected abstract doDeliver( mail: Mail, context: MailDeliverContext, @@ -226,12 +215,13 @@ export abstract class MailDeliverer { return await this.deliver({ mail: new Mail(rawMail) }); } - async deliver( - options: { mail: Mail; recipients?: string[] }, - ): Promise<MailDeliverResult> { - log.info(`Begin to deliver mail via ${this.name}...`); + async deliver(options: { + mail: Mail; + recipients?: string[]; + }): Promise<MailDeliverResult> { + this.logger.info(`Begin to deliver mail via ${this.name}...`); - const context = new MailDeliverContext(options.mail); + const context = new MailDeliverContext(this.logger, options.mail); options.recipients?.forEach((r) => context.recipients.add(r)); for (const hook of this.preHooks) { @@ -244,8 +234,8 @@ export abstract class MailDeliverer { await hook.callback(context); } - log.info("Deliver result:"); - log.info(context.result); + context.logger.info("Deliver result:"); + context.logger.info(context.result); if (context.result.hasError()) { throw new Error("Mail failed to deliver."); @@ -261,7 +251,7 @@ export abstract class SyncMailDeliverer extends MailDeliverer { override async deliver( options: { mail: Mail; recipients?: string[] }, ): Promise<MailDeliverResult> { - log.info("The mail deliverer is sync. Wait for last delivering done..."); + this.logger.info("The mail deliverer is sync. Wait for last delivering done..."); await this.#last; const result = super.deliver(options); this.#last = result.then(() => {}, () => {}); @@ -270,17 +260,24 @@ export abstract class SyncMailDeliverer extends MailDeliverer { } export class RecipientFromHeadersHook implements MailDeliverHook { + constructor(public mailDomain: string) {} + callback(context: MailDeliverContext) { if (context.recipients.size !== 0) { - log.warn( + context.logger.warn( "Recipients are already filled. Won't set them with ones in headers.", ); } else { - context.mail.startSimpleParse().sections().headers().recipients({ - domain: config.get("mailDomain"), - }).forEach((r) => context.recipients.add(r)); - - log.info( + context.mail + .startSimpleParse(context.logger) + .sections() + .headers() + .recipients({ + domain: this.mailDomain, + }) + .forEach((r) => context.recipients.add(r)); + + context.logger.info( "Recipients found from mail headers: ", [...context.recipients].join(" "), ); @@ -294,7 +291,7 @@ export class FallbackRecipientHook implements MailDeliverHook { callback(context: MailDeliverContext) { if (context.recipients.size === 0) { - log.info( + context.logger.info( "No recipients, fill with fallback: ", [...this.fallback].join(" "), ); @@ -311,10 +308,10 @@ export class AliasRecipientMailHook implements MailDeliverHook { this.#aliasFile = aliasFile; } - async #parseAliasFile(): Promise<Map<string, string>> { + async #parseAliasFile(logger: Logger): Promise<Map<string, string>> { const result = new Map(); if ((await Deno.stat(this.#aliasFile)).isFile) { - log.info(`Found recipients alias file: ${this.#aliasFile}.`); + logger.info(`Found recipients alias file: ${this.#aliasFile}.`); const text = await Deno.readTextFile(this.#aliasFile); const csv = parse(text); for (const [real, ...aliases] of csv) { @@ -325,11 +322,11 @@ export class AliasRecipientMailHook implements MailDeliverHook { } async callback(context: MailDeliverContext) { - const aliases = await this.#parseAliasFile(); + const aliases = await this.#parseAliasFile(context.logger); for (const recipient of [...context.recipients]) { const realRecipients = aliases.get(recipient); if (realRecipients != null) { - log.info( + context.logger.info( `Recipient alias resolved: ${recipient} => ${realRecipients}.`, ); context.recipients.delete(recipient); diff --git a/services/docker/mail-server/app/main.bash b/services/docker/mail-server/app/main.bash index f57d254..14900d7 100755 --- a/services/docker/mail-server/app/main.bash +++ b/services/docker/mail-server/app/main.bash @@ -7,7 +7,5 @@ die() { exit 1 } -/app/crupest-relay init || die "crupest-relay failed to init." - /app/crupest-relay real-serve & /dovecot/sbin/dovecot -F |