diff options
Diffstat (limited to 'deno/mail-relay')
-rw-r--r-- | deno/mail-relay/app.ts | 84 | ||||
-rw-r--r-- | deno/mail-relay/aws/app.ts | 297 | ||||
-rw-r--r-- | deno/mail-relay/aws/deliver.ts | 60 | ||||
-rw-r--r-- | deno/mail-relay/aws/fetch.ts | 127 | ||||
-rw-r--r-- | deno/mail-relay/aws/mail.ts | 49 | ||||
-rw-r--r-- | deno/mail-relay/db.test.ts | 23 | ||||
-rw-r--r-- | deno/mail-relay/db.ts | 146 | ||||
-rw-r--r-- | deno/mail-relay/deno.json | 18 | ||||
-rw-r--r-- | deno/mail-relay/dovecot.ts | 99 | ||||
-rw-r--r-- | deno/mail-relay/dumb-smtp-server.ts | 130 | ||||
-rw-r--r-- | deno/mail-relay/mail.test.ts | 126 | ||||
-rw-r--r-- | deno/mail-relay/mail.ts | 330 |
12 files changed, 1489 insertions, 0 deletions
diff --git a/deno/mail-relay/app.ts b/deno/mail-relay/app.ts new file mode 100644 index 0000000..eeffc12 --- /dev/null +++ b/deno/mail-relay/app.ts @@ -0,0 +1,84 @@ +import { Hono } from "hono"; +import { logger as honoLogger } from "hono/logger"; + +import { LogFileProvider } from "@crupest/base/log"; + +import { + AliasRecipientMailHook, + FallbackRecipientHook, + MailDeliverer, + RecipientFromHeadersHook, +} from "./mail.ts"; +import { DovecotMailDeliverer } from "./dovecot.ts"; +import { DumbSmtpServer } from "./dumb-smtp-server.ts"; + +export function createInbound( + logFileProvider: LogFileProvider, + { + fallback, + mailDomain, + aliasFile, + ldaPath, + }: { + fallback: string[]; + mailDomain: string; + aliasFile: string; + ldaPath: string; + }, +) { + const deliverer = new DovecotMailDeliverer(logFileProvider, ldaPath); + deliverer.preHooks.push( + new RecipientFromHeadersHook(mailDomain), + new FallbackRecipientHook(new Set(fallback)), + new AliasRecipientMailHook(aliasFile), + ); + return deliverer; +} + +export function createHono(outbound: MailDeliverer, inbound: MailDeliverer) { + const hono = new Hono(); + + hono.onError((err, c) => { + console.error("Hono handler throws an error.", err); + return c.json({ msg: "Server error, check its log." }, 500); + }); + hono.use(honoLogger()); + hono.post("/send/raw", async (context) => { + const body = await context.req.text(); + if (body.trim().length === 0) { + return context.json({ msg: "Can't send an empty mail." }, 400); + } else { + const result = await outbound.deliverRaw(body); + return context.json({ + awsMessageId: result.awsMessageId, + }); + } + }); + hono.post("/receive/raw", async (context) => { + await inbound.deliverRaw(await context.req.text()); + return context.json({ msg: "Done!" }); + }); + + return hono; +} + +export function createSmtp(outbound: MailDeliverer) { + return new DumbSmtpServer(outbound); +} + +export async function sendMail(port: number) { + const decoder = new TextDecoder(); + let text = ""; + for await (const chunk of Deno.stdin.readable) { + text += decoder.decode(chunk); + } + + const res = await fetch(`http://127.0.0.1:${port}/send/raw`, { + method: "post", + body: text, + }); + const fn = res.ok ? "info" : "error"; + console[fn](res); + console[fn](await res.text()); + if (!res.ok) Deno.exit(-1); +} diff --git a/deno/mail-relay/aws/app.ts b/deno/mail-relay/aws/app.ts new file mode 100644 index 0000000..cb275ae --- /dev/null +++ b/deno/mail-relay/aws/app.ts @@ -0,0 +1,297 @@ +import { join } from "@std/path"; +import { z } from "zod"; +import { Hono } from "hono"; +import { zValidator } from "@hono/zod-validator"; +import { FetchHttpHandler } from "@smithy/fetch-http-handler"; +// @ts-types="npm:@types/yargs" +import yargs from "yargs"; + +import { LogFileProvider } from "@crupest/base/log"; +import { ConfigDefinition, ConfigProvider } from "@crupest/base/config"; +import { CronTask } from "@crupest/base/cron"; + +import { DbService } from "../db.ts"; +import { Mail } from "../mail.ts"; +import { + AwsMailMessageIdRewriteHook, + AwsMailMessageIdSaveHook, +} from "./mail.ts"; +import { AwsMailDeliverer } from "./deliver.ts"; +import { AwsMailFetcher, AwsS3MailConsumer } from "./fetch.ts"; +import { createHono, createInbound, createSmtp, sendMail } from "../app.ts"; + +const PREFIX = "crupest-mail-server"; +const CONFIG_DEFINITIONS = { + dataPath: { + description: "Path to save app persistent data.", + default: ".", + }, + mailDomain: { + description: + "The part after `@` of an address. Used to determine local recipients.", + }, + httpHost: { + description: "Listening address for http server.", + default: "0.0.0.0", + }, + httpPort: { description: "Listening port for http server.", default: "2345" }, + smtpHost: { + description: "Listening address for dumb smtp server.", + default: "127.0.0.1", + }, + smtpPort: { + description: "Listening port for dumb smtp server.", + default: "2346", + }, + ldaPath: { + description: "full path of lda executable", + default: "/dovecot/libexec/dovecot/dovecot-lda", + }, + inboundFallback: { + description: "comma separated addresses used as fallback recipients", + default: "", + }, + awsInboundPath: { + description: "(random set) path for aws sns", + }, + awsInboundKey: { + description: "(random set) http header Authorization for aws sns", + }, + awsRegion: { + description: "aws region", + }, + awsUser: { + description: "aws access key id", + }, + awsPassword: { + description: "aws secret access key", + secret: true, + }, + awsMailBucket: { + description: "aws s3 bucket saving raw mails", + secret: true, + }, +} as const satisfies ConfigDefinition; + +function createAwsOptions({ + user, + password, + region, +}: { + user: string; + password: string; + region: string; +}) { + return { + credentials: () => + Promise.resolve({ + accessKeyId: user, + secretAccessKey: password, + }), + requestHandler: new FetchHttpHandler(), + region, + }; +} + +function createOutbound( + awsOptions: ReturnType<typeof createAwsOptions>, + db: DbService, +) { + const deliverer = new AwsMailDeliverer(awsOptions); + deliverer.preHooks.push( + new AwsMailMessageIdRewriteHook(db.messageIdToAws.bind(db)), + ); + deliverer.postHooks.push( + new AwsMailMessageIdSaveHook((original, aws) => + db.addMessageIdMap({ message_id: original, aws_message_id: aws }).then() + ), + ); + return deliverer; +} + +function setupAwsHono( + hono: Hono, + options: { + path: string; + auth: string; + callback: (s3Key: string, recipients?: string[]) => Promise<void>; + }, +) { + hono.post( + `/${options.path}`, + async (ctx, next) => { + const auth = ctx.req.header("Authorization"); + if (auth !== options.auth) { + return ctx.json({ msg: "Bad auth!" }, 403); + } + await next(); + }, + zValidator( + "json", + z.object({ + key: z.string(), + recipients: z.optional(z.array(z.string())), + }), + ), + async (ctx) => { + const { key, recipients } = ctx.req.valid("json"); + await options.callback(key, recipients); + return ctx.json({ msg: "Done!" }); + }, + ); +} + +function createCron(fetcher: AwsMailFetcher, consumer: AwsS3MailConsumer) { + return new CronTask({ + name: "live-mail-recycler", + interval: 6 * 3600 * 1000, + callback: () => { + return fetcher.recycleLiveMails(consumer); + }, + startNow: true, + }); +} + +function createBaseServices() { + const config = new ConfigProvider(PREFIX, CONFIG_DEFINITIONS); + Deno.mkdirSync(config.get("dataPath"), { recursive: true }); + const logFileProvider = new LogFileProvider( + join(config.get("dataPath"), "log"), + ); + return { config, logFileProvider }; +} + +function createAwsFetchOnlyServices() { + const services = createBaseServices(); + const { config } = services; + + const awsOptions = createAwsOptions({ + user: config.get("awsUser"), + password: config.get("awsPassword"), + region: config.get("awsRegion"), + }); + const fetcher = new AwsMailFetcher(awsOptions, config.get("awsMailBucket")); + + return { ...services, awsOptions, fetcher }; +} + +function createAwsRecycleOnlyServices() { + const services = createAwsFetchOnlyServices(); + const { config, logFileProvider } = services; + + const inbound = createInbound(logFileProvider, { + fallback: config.getList("inboundFallback"), + ldaPath: config.get("ldaPath"), + aliasFile: join(config.get("dataPath"), "aliases.csv"), + mailDomain: config.get("mailDomain"), + }); + const recycler = (rawMail: string, _: unknown): Promise<void> => + inbound.deliver({ mail: new Mail(rawMail) }).then(); + + return { ...services, inbound, recycler }; +} +function createAwsServices() { + const services = createAwsRecycleOnlyServices(); + const { config, awsOptions } = services; + + const dbService = new DbService(join(config.get("dataPath"), "db.sqlite")); + const outbound = createOutbound(awsOptions, dbService); + + return { ...services, dbService, outbound }; +} + +function createServerServices() { + const services = createAwsServices(); + const { config, outbound, inbound, fetcher } = services; + + const smtp = createSmtp(outbound); + const hono = createHono(outbound, inbound); + setupAwsHono(hono, { + path: config.get("awsInboundPath"), + auth: config.get("awsInboundKey"), + callback: (s3Key, recipients) => { + return fetcher.consumeS3Mail( + s3Key, + (rawMail, _) => + inbound.deliver({ mail: new Mail(rawMail), recipients }).then(), + ); + }, + }); + + return { ...services, smtp, hono }; +} + +function serve(cron: boolean = false) { + const { config, fetcher, recycler, smtp, hono } = createServerServices(); + smtp.serve({ + hostname: config.get("smtpHost"), + port: config.getInt("smtpPort"), + }); + Deno.serve( + { + hostname: config.get("httpHost"), + port: config.getInt("httpPort"), + }, + hono.fetch, + ); + + if (cron) { + createCron(fetcher, recycler); + } +} + +async function listLives() { + const { fetcher } = createAwsFetchOnlyServices(); + const liveMails = await fetcher.listLiveMails(); + console.info(`Total ${liveMails.length}:`); + if (liveMails.length !== 0) { + console.info(liveMails.join("\n")); + } +} + +async function recycleLives() { + const { fetcher, recycler } = createAwsRecycleOnlyServices(); + await fetcher.recycleLiveMails(recycler); +} + +if (import.meta.main) { + await yargs(Deno.args) + .scriptName("mail-relay") + .command({ + command: "sendmail", + describe: "send mail via this server's endpoint", + handler: async (_argv) => { + const { config } = createBaseServices(); + await sendMail(config.getInt("httpPort")); + }, + }) + .command({ + command: "live", + describe: "work with live mails", + builder: (builder) => { + return builder + .command({ + command: "list", + describe: "list live mails", + handler: listLives, + }) + .command({ + command: "recycle", + describe: "recycle all live mails", + handler: recycleLives, + }) + .demandCommand(1, "One command must be specified."); + }, + handler: () => {}, + }) + .command({ + command: "serve", + describe: "start the http and smtp servers", + builder: (builder) => builder.option("real", { type: "boolean" }), + handler: (argv) => serve(argv.real), + }) + .demandCommand(1, "One command must be specified.") + .help() + .strict() + .parse(); +} diff --git a/deno/mail-relay/aws/deliver.ts b/deno/mail-relay/aws/deliver.ts new file mode 100644 index 0000000..4dd4b3a --- /dev/null +++ b/deno/mail-relay/aws/deliver.ts @@ -0,0 +1,60 @@ +import { + SendEmailCommand, + SESv2Client, + SESv2ClientConfig, +} from "@aws-sdk/client-sesv2"; + +import { Mail, MailDeliverContext, SyncMailDeliverer } from "../mail.ts"; + +declare module "../mail.ts" { + interface MailDeliverResult { + awsMessageId?: string; + } +} + +export class AwsMailDeliverer extends SyncMailDeliverer { + readonly name = "aws"; + readonly #aws; + readonly #ses; + + constructor(aws: SESv2ClientConfig) { + super(); + this.#aws = aws; + this.#ses = new SESv2Client(aws); + } + + protected override async doDeliver( + mail: Mail, + context: MailDeliverContext, + ): Promise<void> { + console.info("Begin to call aws send-email api..."); + + try { + const sendCommand = new SendEmailCommand({ + Content: { + Raw: { Data: mail.toUtf8Bytes() }, + }, + }); + + const res = await this.#ses.send(sendCommand); + if (res.MessageId == null) { + console.warn("Aws send-email returns no message id."); + } else { + context.result.awsMessageId = + `${res.MessageId}@${this.#aws.region}.amazonses.com`; + } + + context.result.recipients.set("*", { + kind: "done", + message: + `Successfully called aws send-email, message id ${context.result.awsMessageId}.`, + }); + } catch (cause) { + context.result.recipients.set("*", { + kind: "fail", + message: "An error was thrown when calling aws send-email." + cause, + cause, + }); + } + } +} diff --git a/deno/mail-relay/aws/fetch.ts b/deno/mail-relay/aws/fetch.ts new file mode 100644 index 0000000..9278e63 --- /dev/null +++ b/deno/mail-relay/aws/fetch.ts @@ -0,0 +1,127 @@ +import { + CopyObjectCommand, + DeleteObjectCommand, + GetObjectCommand, + ListObjectsV2Command, + S3Client, + S3ClientConfig, +} from "@aws-sdk/client-s3"; + +import { toFileNameString } from "@crupest/base"; + +import { Mail } from "../mail.ts"; + +async function s3MoveObject( + client: S3Client, + bucket: string, + path: string, + newPath: string, +): Promise<void> { + const copyCommand = new CopyObjectCommand({ + Bucket: bucket, + Key: newPath, + CopySource: `${bucket}/${path}`, + }); + await client.send(copyCommand); + + const deleteCommand = new DeleteObjectCommand({ + Bucket: bucket, + Key: path, + }); + await client.send(deleteCommand); +} + +const AWS_SES_S3_SETUP_TAG = "AMAZON_SES_SETUP_NOTIFICATION"; + +export type AwsS3MailConsumer = ( + rawMail: string, + s3Key: string, +) => Promise<void>; + +export class AwsMailFetcher { + readonly #livePrefix = "mail/live/"; + readonly #archivePrefix = "mail/archive/"; + readonly #s3; + readonly #bucket; + + constructor(aws: S3ClientConfig, bucket: string) { + this.#s3 = new S3Client(aws); + this.#bucket = bucket; + } + + async listLiveMails(): Promise<string[]> { + console.info("Begin to retrieve live mails."); + + const listCommand = new ListObjectsV2Command({ + Bucket: this.#bucket, + Prefix: this.#livePrefix, + }); + const res = await this.#s3.send(listCommand); + + if (res.Contents == null) { + console.warn("Listing live mails in S3 returns null Content."); + return []; + } + + const result: string[] = []; + for (const object of res.Contents) { + if (object.Key == null) { + console.warn("Listing live mails in S3 returns an object with no Key."); + continue; + } + + if (object.Key.endsWith(AWS_SES_S3_SETUP_TAG)) continue; + + result.push(object.Key.slice(this.#livePrefix.length)); + } + return result; + } + + async consumeS3Mail(s3Key: string, consumer: AwsS3MailConsumer) { + console.info(`Begin to consume s3 mail ${s3Key} ...`); + + console.info(`Fetching s3 mail ${s3Key}...`); + const mailPath = `${this.#livePrefix}${s3Key}`; + const command = new GetObjectCommand({ + Bucket: this.#bucket, + Key: mailPath, + }); + const res = await this.#s3.send(command); + + if (res.Body == null) { + throw new Error("S3 mail returns a null body."); + } + + const rawMail = await res.Body.transformToString(); + console.info(`Done fetching s3 mail ${s3Key}.`); + + console.info(`Calling consumer...`); + await consumer(rawMail, s3Key); + console.info(`Done consuming s3 mail ${s3Key}.`); + + const date = new Mail(rawMail) + .startSimpleParse() + .sections() + .headers() + .date(); + const dateString = date != null + ? toFileNameString(date, true) + : "invalid-date"; + const newPath = `${this.#archivePrefix}${dateString}/${s3Key}`; + + console.info(`Archiving s3 mail ${s3Key} to ${newPath}...`); + await s3MoveObject(this.#s3, this.#bucket, mailPath, newPath); + console.info(`Done archiving s3 mail ${s3Key}.`); + + console.info(`Done consuming s3 mail ${s3Key}.`); + } + + async recycleLiveMails(consumer: AwsS3MailConsumer) { + console.info("Begin to recycle live mails..."); + const mails = await this.listLiveMails(); + console.info(`Found ${mails.length} live mails`); + for (const s3Key of mails) { + await this.consumeS3Mail(s3Key, consumer); + } + } +} diff --git a/deno/mail-relay/aws/mail.ts b/deno/mail-relay/aws/mail.ts new file mode 100644 index 0000000..cc05d23 --- /dev/null +++ b/deno/mail-relay/aws/mail.ts @@ -0,0 +1,49 @@ +import { MailDeliverContext, MailDeliverHook } from "../mail.ts"; + +export class AwsMailMessageIdRewriteHook implements MailDeliverHook { + readonly #lookup; + + constructor(lookup: (origin: string) => Promise<string | null>) { + this.#lookup = lookup; + } + + async callback(context: MailDeliverContext): Promise<void> { + console.info("Rewrite message ids..."); + const addresses = context.mail.simpleFindAllAddresses(); + console.info(`Addresses found in mail: ${addresses.join(", ")}.`); + for (const address of addresses) { + const awsMessageId = await this.#lookup(address); + if (awsMessageId != null && awsMessageId.length !== 0) { + console.info(`Rewrite ${address} to ${awsMessageId}.`); + context.mail.raw = context.mail.raw.replaceAll(address, awsMessageId); + } + } + console.info("Done rewrite message ids."); + } +} + +export class AwsMailMessageIdSaveHook implements MailDeliverHook { + readonly #record; + + constructor(record: (original: string, aws: string) => Promise<void>) { + this.#record = record; + } + + async callback(context: MailDeliverContext): Promise<void> { + console.info("Save aws message ids..."); + const messageId = context.mail + .startSimpleParse() + .sections() + .headers() + .messageId(); + if (messageId == null) { + console.info("Original mail does not have message id. Skip saving."); + return; + } + if (context.result.awsMessageId != null) { + console.info(`Saving ${messageId} => ${context.result.awsMessageId}.`); + await this.#record(messageId, context.result.awsMessageId); + } + console.info("Done save message ids."); + } +} diff --git a/deno/mail-relay/db.test.ts b/deno/mail-relay/db.test.ts new file mode 100644 index 0000000..60035c4 --- /dev/null +++ b/deno/mail-relay/db.test.ts @@ -0,0 +1,23 @@ +import { describe, it } from "@std/testing/bdd"; +import { expect } from "@std/expect/expect"; + +import { DbService } from "./db.ts"; + +describe("DbService", () => { + const mockRow = { + message_id: "mock-message-id@mock.mock", + aws_message_id: "mock-aws-message-id@mock.mock", + }; + + it("works", async () => { + const db = new DbService(":memory:"); + await db.migrate(); + await db.addMessageIdMap(mockRow); + expect(await db.messageIdToAws(mockRow.message_id)).toBe( + mockRow.aws_message_id, + ); + expect(await db.messageIdFromAws(mockRow.aws_message_id)).toBe( + mockRow.message_id, + ); + }); +}); diff --git a/deno/mail-relay/db.ts b/deno/mail-relay/db.ts new file mode 100644 index 0000000..062700b --- /dev/null +++ b/deno/mail-relay/db.ts @@ -0,0 +1,146 @@ +import { + Generated, + Insertable, + Kysely, + Migration, + Migrator, + SqliteDatabase, + SqliteDialect, + SqliteStatement, +} from "kysely"; +import * as sqlite from "@db/sqlite"; + +class SqliteStatementAdapter implements SqliteStatement { + constructor(public readonly stmt: sqlite.Statement) {} + + get reader(): boolean { + return this.stmt.columnNames().length >= 1; + } + + all(parameters: readonly unknown[]): unknown[] { + return this.stmt.all(...(parameters as sqlite.BindValue[])); + } + + iterate(parameters: readonly unknown[]): IterableIterator<unknown> { + return this.stmt.iter(...(parameters as sqlite.BindValue[])); + } + + run(parameters: readonly unknown[]): { + changes: number | bigint; + lastInsertRowid: number | bigint; + } { + const { db } = this.stmt; + const totalChangesBefore = db.totalChanges; + const changes = this.stmt.run(...(parameters as sqlite.BindValue[])); + return { + changes: totalChangesBefore === db.totalChanges ? 0 : changes, + lastInsertRowid: db.lastInsertRowId, + }; + } +} + +class SqliteDatabaseAdapter implements SqliteDatabase { + constructor(public readonly db: sqlite.Database) {} + + prepare(sql: string): SqliteStatementAdapter { + return new SqliteStatementAdapter(this.db.prepare(sql)); + } + + close(): void { + this.db.close(); + } +} + +export class DbError extends Error {} + +interface AwsMessageIdMapTable { + id: Generated<number>; + message_id: string; + aws_message_id: string; +} + +interface Database { + aws_message_id_map: AwsMessageIdMapTable; +} + +const migrations: Record<string, Migration> = { + "0001-init": { + // deno-lint-ignore no-explicit-any + async up(db: Kysely<any>): Promise<void> { + await db.schema + .createTable("aws_message_id_map") + .addColumn("id", "integer", (col) => col.primaryKey().autoIncrement()) + .addColumn("message_id", "text", (col) => col.notNull().unique()) + .addColumn("aws_message_id", "text", (col) => col.notNull().unique()) + .execute(); + + for (const column of ["message_id", "aws_message_id"]) { + await db.schema + .createIndex(`aws_message_id_map_${column}`) + .on("aws_message_id_map") + .column(column) + .execute(); + } + }, + + // deno-lint-ignore no-explicit-any + async down(db: Kysely<any>): Promise<void> { + await db.schema.dropTable("aws_message_id_map").execute(); + }, + }, +}; + +export class DbService { + #db; + #kysely; + #migrator; + + constructor(public readonly path: string) { + this.#db = new sqlite.Database(path); + this.#kysely = new Kysely<Database>({ + dialect: new SqliteDialect({ + database: new SqliteDatabaseAdapter(this.#db), + }), + }); + this.#migrator = new Migrator({ + db: this.#kysely, + provider: { + getMigrations(): Promise<Record<string, Migration>> { + return Promise.resolve(migrations); + }, + }, + }); + } + + async migrate(): Promise<void> { + await this.#migrator.migrateToLatest(); + } + + async addMessageIdMap( + mail: Insertable<AwsMessageIdMapTable>, + ): Promise<number> { + const inserted = await this.#kysely + .insertInto("aws_message_id_map") + .values(mail) + .executeTakeFirstOrThrow(); + return Number(inserted.insertId!); + } + + async messageIdToAws(messageId: string): Promise<string | null> { + const row = await this.#kysely + .selectFrom("aws_message_id_map") + .where("message_id", "=", messageId) + .select("aws_message_id") + .executeTakeFirst(); + return row?.aws_message_id ?? null; + } + + async messageIdFromAws(awsMessageId: string): Promise<string | null> { + const row = await this.#kysely + .selectFrom("aws_message_id_map") + .where("aws_message_id", "=", awsMessageId) + .select("message_id") + .executeTakeFirst(); + return row?.message_id ?? null; + } +} diff --git a/deno/mail-relay/deno.json b/deno/mail-relay/deno.json new file mode 100644 index 0000000..9105747 --- /dev/null +++ b/deno/mail-relay/deno.json @@ -0,0 +1,18 @@ +{ + "version": "0.1.0", + "tasks": { + "run": "deno run -A aws/app.ts", + "compile": "deno compile -o out/crupest-relay -A aws/app.ts" + }, + "imports": { + "@aws-sdk/client-s3": "npm:@aws-sdk/client-s3@^3.821.0", + "@aws-sdk/client-sesv2": "npm:@aws-sdk/client-sesv2@^3.821.0", + "@db/sqlite": "jsr:@db/sqlite@^0.12.0", + "@hono/zod-validator": "npm:@hono/zod-validator@^0.7.0", + "@smithy/fetch-http-handler": "npm:@smithy/fetch-http-handler@^5.0.4", + "email-addresses": "npm:email-addresses@^5.0.0", + "hono": "npm:hono@^4.7.11", + "kysely": "npm:kysely@^0.28.2", + "zod": "npm:zod@^3.25.48" + } +} diff --git a/deno/mail-relay/dovecot.ts b/deno/mail-relay/dovecot.ts new file mode 100644 index 0000000..bace225 --- /dev/null +++ b/deno/mail-relay/dovecot.ts @@ -0,0 +1,99 @@ +import { basename } from "@std/path"; + +import { LogFileProvider } from "@crupest/base/log"; + +import { Mail, MailDeliverContext, MailDeliverer } from "./mail.ts"; + +export class DovecotMailDeliverer extends MailDeliverer { + readonly name = "dovecot"; + readonly #logFileProvider; + readonly #ldaPath; + + constructor(logFileProvider: LogFileProvider, ldaPath: string) { + super(); + this.#logFileProvider = logFileProvider; + this.#ldaPath = ldaPath; + } + + protected override async doDeliver( + mail: Mail, + context: MailDeliverContext, + ): Promise<void> { + const ldaPath = this.#ldaPath; + const ldaBinName = basename(ldaPath); + const utf8Stream = mail.toUtf8Bytes(); + + const recipients = [...context.recipients]; + + if (recipients.length === 0) { + context.result.message = + "Failed to deliver to dovecot, no recipients are specified."; + return; + } + + console.info(`Deliver to dovecot users: ${recipients.join(", ")}.`); + + for (const recipient of recipients) { + try { + const commandArgs = ["-d", recipient]; + console.info(`Run ${ldaBinName} ${commandArgs.join(" ")}...`); + + const ldaCommand = new Deno.Command(ldaPath, { + args: commandArgs, + stdin: "piped", + stdout: "piped", + stderr: "piped", + }); + + const ldaProcess = ldaCommand.spawn(); + using logFiles = await this.#logFileProvider + .createExternalLogStreamsForProgram( + ldaBinName, + ); + ldaProcess.stdout.pipeTo(logFiles.stdout); + ldaProcess.stderr.pipeTo(logFiles.stderr); + + const stdinWriter = ldaProcess.stdin.getWriter(); + await stdinWriter.write(utf8Stream); + await stdinWriter.close(); + + const status = await ldaProcess.status; + + if (status.success) { + context.result.recipients.set(recipient, { + kind: "done", + message: `${ldaBinName} exited with success.`, + }); + } else { + let message = `${ldaBinName} exited with error code ${status.code}`; + + if (status.signal != null) { + message += ` (signal ${status.signal})`; + } + + // https://doc.dovecot.org/main/core/man/dovecot-lda.1.html + switch (status.code) { + case 67: + message += ", recipient user not known"; + break; + case 75: + message += ", temporary error"; + break; + } + + message += "."; + + context.result.recipients.set(recipient, { kind: "fail", message }); + } + } catch (cause) { + context.result.recipients.set(recipient, { + kind: "fail", + message: "An error is thrown when running lda: " + cause, + cause, + }); + } + } + + console.info("Done handling all recipients."); + } +} diff --git a/deno/mail-relay/dumb-smtp-server.ts b/deno/mail-relay/dumb-smtp-server.ts new file mode 100644 index 0000000..ac7069c --- /dev/null +++ b/deno/mail-relay/dumb-smtp-server.ts @@ -0,0 +1,130 @@ +import { MailDeliverer } from "./mail.ts"; + +const CRLF = "\r\n"; + +function createResponses(host: string, port: number | string) { + const serverName = `[${host}]:${port}`; + return { + serverName, + READY: `220 ${serverName} SMTP Ready`, + EHLO: `250 ${serverName}`, + MAIL: "250 2.1.0 Sender OK", + RCPT: "250 2.1.5 Recipient OK", + DATA: "354 Start mail input; end with <CRLF>.<CRLF>", + QUIT: `211 2.0.0 ${serverName} closing connection`, + INVALID: "500 5.5.1 Error: command not recognized", + } as const; +} + +const LOG_TAG = "[dumb-smtp]"; + +export class DumbSmtpServer { + #deliverer; + #responses: ReturnType<typeof createResponses> = createResponses( + "invalid", + "invalid", + ); + + constructor(deliverer: MailDeliverer) { + this.#deliverer = deliverer; + } + + async #handleConnection(conn: Deno.Conn) { + using disposeStack = new DisposableStack(); + disposeStack.defer(() => { + console.info(LOG_TAG, "Close session's tcp connection."); + conn.close(); + }); + + console.info(LOG_TAG, "New session's tcp connection established."); + + const writer = conn.writable.getWriter(); + disposeStack.defer(() => writer.releaseLock()); + const reader = conn.readable.getReader(); + disposeStack.defer(() => reader.releaseLock()); + + const [decoder, encoder] = [new TextDecoder(), new TextEncoder()]; + const decode = (data: Uint8Array) => decoder.decode(data); + const send = async (s: string) => { + console.info(LOG_TAG, "Send line: " + s); + await writer.write(encoder.encode(s + CRLF)); + }; + + let buffer: string = ""; + let rawMail: string | null = null; + + await send(this.#responses["READY"]); + + while (true) { + const { value, done } = await reader.read(); + if (done) break; + + buffer += decode(value); + + while (true) { + const eolPos = buffer.indexOf(CRLF); + if (eolPos === -1) break; + + const line = buffer.slice(0, eolPos); + buffer = buffer.slice(eolPos + CRLF.length); + + if (rawMail == null) { + console.info(LOG_TAG, "Received line: " + line); + const upperLine = line.toUpperCase(); + if (upperLine.startsWith("EHLO") || upperLine.startsWith("HELO")) { + await send(this.#responses["EHLO"]); + } else if (upperLine.startsWith("MAIL FROM:")) { + await send(this.#responses["MAIL"]); + } else if (upperLine.startsWith("RCPT TO:")) { + await send(this.#responses["RCPT"]); + } else if (upperLine === "DATA") { + await send(this.#responses["DATA"]); + console.info(LOG_TAG, "Begin to receive mail data..."); + rawMail = ""; + } else if (upperLine === "QUIT") { + await send(this.#responses["QUIT"]); + return; + } else { + console.warn(LOG_TAG, "Unrecognized command from client: " + line); + await send(this.#responses["INVALID"]); + return; + } + } else { + if (line === ".") { + try { + console.info(LOG_TAG, "Mail data Received, begin to relay..."); + const { message } = await this.#deliverer.deliverRaw(rawMail); + await send(`250 2.6.0 ${message}`); + rawMail = null; + console.info(LOG_TAG, "Relay succeeded."); + } catch (err) { + console.error(LOG_TAG, "Relay failed.", err); + await send("554 5.3.0 Error: check server log"); + return; + } + } else { + const dataLine = line.startsWith("..") ? line.slice(1) : line; + rawMail += dataLine + CRLF; + } + } + } + } + } + + async serve(options: { hostname: string; port: number }) { + const listener = Deno.listen(options); + this.#responses = createResponses(options.hostname, options.port); + console.info( + LOG_TAG, + `Dumb SMTP server starts to listen on ${this.#responses.serverName}.`, + ); + + for await (const conn of listener) { + try { + await this.#handleConnection(conn); + } catch (cause) { + console.error(LOG_TAG, "Tcp connection throws an error.", cause); + } + } + } +} diff --git a/deno/mail-relay/mail.test.ts b/deno/mail-relay/mail.test.ts new file mode 100644 index 0000000..cd0c38d --- /dev/null +++ b/deno/mail-relay/mail.test.ts @@ -0,0 +1,126 @@ +import { describe, it } from "@std/testing/bdd"; +import { expect, fn } from "@std/expect"; + +import { Mail, MailDeliverContext, MailDeliverer } from "./mail.ts"; + +const mockDate = "Fri, 02 May 2025 08:33:02 +0000"; +const mockMessageId = "mock-message-id@from.mock"; +const mockMessageId2 = "mock-message-id-2@from.mock"; +const mockFromAddress = "mock@from.mock"; +const mockCcAddress = "mock@cc.mock"; +const mockBodyStr = `This is body content. +Line 2 ${mockMessageId2} + +Line 4`; +const mockHeaders = [ + ["Content-Disposition", "inline"], + ["Content-Transfer-Encoding", "quoted-printable"], + ["MIME-Version", "1.0"], + ["X-Mailer", "MIME-tools 5.509 (Entity 5.509)"], + ["Content-Type", "text/plain; charset=utf-8"], + ["From", `"Mock From" <${mockFromAddress}>`], + [ + "To", + `"John \\"Big\\" Doe" <john@example.com>, "Alice (Work)" <alice+work@example.com>, + undisclosed-recipients:;, "Group: Team" <team@company.com>, + "Escaped, Name" <escape@test.com>, just@email.com, + "Comment (This is valid)" <comment@domain.net>, + "Odd @Chars" <weird!#$%'*+-/=?^_\`{|}~@char-test.com>, + "Non-ASCII 用户" <user@例子.中国>, + admin@[192.168.1.1]`, + ], + ["CC", `Mock CC <${mockCcAddress}>`], + ["Subject", "A very long mock\n subject"], + ["Message-ID", `<${mockMessageId}>`], + ["Date", mockDate], +]; +const mockHeaderStr = mockHeaders.map((h) => h[0] + ": " + h[1]).join("\n"); +const mockMailStr = mockHeaderStr + "\n\n" + mockBodyStr; +const mockCrlfMailStr = mockMailStr.replaceAll("\n", "\r\n"); +const mockToAddresses = [ + "john@example.com", + "alice+work@example.com", + "team@company.com", + "escape@test.com", + "just@email.com", + "comment@domain.net", + "weird!#$%'*+-/=?^_`{|}~@char-test.com", + "user@例子.中国", + "admin@[192.168.1.1]", +]; + +describe("Mail", () => { + it("simple parse", () => { + const parsed = new Mail(mockMailStr).startSimpleParse().sections(); + expect(parsed.header).toEqual(mockHeaderStr); + expect(parsed.body).toEqual(mockBodyStr); + expect(parsed.sep).toBe("\n"); + expect(parsed.eol).toBe("\n"); + }); + + it("simple parse crlf", () => { + const parsed = new Mail(mockCrlfMailStr).startSimpleParse().sections(); + expect(parsed.sep).toBe("\r\n"); + expect(parsed.eol).toBe("\r\n"); + }); + + it("simple parse date", () => { + expect( + new Mail(mockMailStr).startSimpleParse().sections().headers().date(), + ).toEqual(new Date(mockDate)); + }); + + it("simple parse headers", () => { + expect( + new Mail(mockMailStr).startSimpleParse().sections().headers().fields, + ).toEqual(mockHeaders.map((h) => [h[0], " " + h[1].replaceAll("\n", "")])); + }); + + it("parse recipients", () => { + const mail = new Mail(mockMailStr); + expect([ + ...mail.startSimpleParse().sections().headers().recipients(), + ]).toEqual([...mockToAddresses, mockCcAddress]); + expect([ + ...mail.startSimpleParse().sections().headers().recipients({ + domain: "example.com", + }), + ]).toEqual( + [...mockToAddresses, mockCcAddress].filter((a) => + a.endsWith("example.com") + ), + ); + }); + + it("find all addresses", () => { + const mail = new Mail(mockMailStr); + expect(mail.simpleFindAllAddresses()).toEqual([ + "mock@from.mock", + "john@example.com", + "alice+work@example.com", + "team@company.com", + "escape@test.com", + "just@email.com", + "comment@domain.net", + "mock@cc.mock", + "mock-message-id@from.mock", + "mock-message-id-2@from.mock", + ]); + }); +}); + +describe("MailDeliverer", () => { + class MockMailDeliverer extends MailDeliverer { + name = "mock"; + override doDeliver = fn((_: Mail, ctx: MailDeliverContext) => { + ctx.result.recipients.set("*", { kind: "done", message: "success" }); + return Promise.resolve(); + }) as MailDeliverer["doDeliver"]; + } + const mockDeliverer = new MockMailDeliverer(); + + it("deliver success", async () => { + await mockDeliverer.deliverRaw(mockMailStr); + expect(mockDeliverer.doDeliver).toHaveBeenCalledTimes(1); + }); +}); diff --git a/deno/mail-relay/mail.ts b/deno/mail-relay/mail.ts new file mode 100644 index 0000000..d6dfe65 --- /dev/null +++ b/deno/mail-relay/mail.ts @@ -0,0 +1,330 @@ +import { encodeBase64 } from "@std/encoding/base64"; +import { parse } from "@std/csv/parse"; +import emailAddresses from "email-addresses"; + +class MailSimpleParseError extends Error {} + +class MailSimpleParsedHeaders { + constructor(public fields: [key: string, value: string][]) {} + + getFirst(fieldKey: string): string | undefined { + for (const [key, value] of this.fields) { + if (key.toLowerCase() === fieldKey.toLowerCase()) return value; + } + return undefined; + } + + messageId(): string | undefined { + const messageIdField = this.getFirst("message-id"); + if (messageIdField == null) return undefined; + + const match = messageIdField.match(/\<(.*?)\>/); + if (match != null) { + return match[1]; + } else { + console.warn("Invalid message-id header of mail: " + messageIdField); + return undefined; + } + } + + date(invalidToUndefined: boolean = true): Date | undefined { + const dateField = this.getFirst("date"); + if (dateField == null) return undefined; + + const date = new Date(dateField); + if (invalidToUndefined && isNaN(date.getTime())) { + console.warn(`Invalid date string (${dateField}) found in header.`); + return undefined; + } + return date; + } + + recipients(options?: { domain?: string; headers?: string[] }): Set<string> { + const domain = options?.domain; + const headers = options?.headers ?? ["to", "cc", "bcc", "x-original-to"]; + const recipients = new Set<string>(); + for (const [key, value] of this.fields) { + if (headers.includes(key.toLowerCase())) { + emailAddresses + .parseAddressList(value) + ?.flatMap((a) => (a.type === "mailbox" ? a : a.addresses)) + ?.forEach(({ address }) => { + if (domain == null || address.endsWith(domain)) { + recipients.add(address); + } + }); + } + } + return recipients; + } +} + +class MailSimpleParsedSections { + header: string; + body: string; + eol: string; + sep: string; + + constructor(raw: string) { + const twoEolMatch = raw.match(/(\r?\n)(\r?\n)/); + if (twoEolMatch == null) { + throw new MailSimpleParseError( + "No header/body section separator (2 successive EOLs) found.", + ); + } + + const [eol, sep] = [twoEolMatch[1], twoEolMatch[2]]; + + if (eol !== sep) { + console.warn("Different EOLs (\\r\\n, \\n) found."); + } + + this.header = raw.slice(0, twoEolMatch.index!); + this.body = raw.slice(twoEolMatch.index! + eol.length + sep.length); + this.eol = eol; + this.sep = sep; + } + + headers(): MailSimpleParsedHeaders { + const headers = [] as [key: string, value: string][]; + + let field: string | null = null; + let lineNumber = 1; + + const handleField = () => { + if (field == null) return; + const sepPos = field.indexOf(":"); + if (sepPos === -1) { + throw new MailSimpleParseError(`No ':' in the header line: ${field}`); + } + headers.push([field.slice(0, sepPos).trim(), field.slice(sepPos + 1)]); + field = null; + }; + + for (const line of this.header.trimEnd().split(/\r?\n|\r/)) { + if (line.match(/^\s/)) { + if (field == null) { + throw new MailSimpleParseError("Header section starts with a space."); + } + field += line; + } else { + handleField(); + field = line; + } + lineNumber += 1; + } + + handleField(); + + return new MailSimpleParsedHeaders(headers); + } +} + +export class Mail { + constructor(public raw: string) {} + + toUtf8Bytes(): Uint8Array { + const utf8Encoder = new TextEncoder(); + return utf8Encoder.encode(this.raw); + } + + toBase64(): string { + return encodeBase64(this.raw); + } + + startSimpleParse() { + return { sections: () => new MailSimpleParsedSections(this.raw) }; + } + + simpleFindAllAddresses(): string[] { + const re = /,?\<?([a-z0-9_'+\-\.]+\@[a-z0-9_'+\-\.]+)\>?,?/gi; + return [...this.raw.matchAll(re)].map((m) => m[1]); + } +} + +export type MailDeliverResultKind = "done" | "fail"; + +export interface MailDeliverRecipientResult { + kind: MailDeliverResultKind; + message: string; + cause?: unknown; +} + +export class MailDeliverResult { + message: string = ""; + recipients: Map<string, MailDeliverRecipientResult> = new Map(); + + constructor(public mail: Mail) {} + + hasError(): boolean { + return ( + this.recipients.size === 0 || + this.recipients.values().some((r) => r.kind !== "done") + ); + } + + [Symbol.for("Deno.customInspect")]() { + return [ + `message: ${this.message}`, + ...this.recipients + .entries() + .map( + ([recipient, result]) => + `${recipient} [${result.kind}]: ${result.message}`, + ), + ].join("\n"); + } +} + +export class MailDeliverContext { + readonly recipients: Set<string> = new Set(); + readonly result; + + constructor(public mail: Mail) { + this.result = new MailDeliverResult(this.mail); + } +} + +export interface MailDeliverHook { + callback(context: MailDeliverContext): Promise<void>; +} + +export abstract class MailDeliverer { + abstract readonly name: string; + preHooks: MailDeliverHook[] = []; + postHooks: MailDeliverHook[] = []; + + protected abstract doDeliver( + mail: Mail, + context: MailDeliverContext, + ): Promise<void>; + + async deliverRaw(rawMail: string) { + return await this.deliver({ mail: new Mail(rawMail) }); + } + + async deliver(options: { + mail: Mail; + recipients?: string[]; + }): Promise<MailDeliverResult> { + console.info(`Begin to deliver mail via ${this.name}...`); + + const context = new MailDeliverContext(options.mail); + options.recipients?.forEach((r) => context.recipients.add(r)); + + for (const hook of this.preHooks) { + await hook.callback(context); + } + + await this.doDeliver(context.mail, context); + + for (const hook of this.postHooks) { + await hook.callback(context); + } + + console.info("Deliver result:"); + console.info(context.result); + + if (context.result.hasError()) { + throw new Error("Mail failed to deliver."); + } + + return context.result; + } +} + +export abstract class SyncMailDeliverer extends MailDeliverer { + #last: Promise<void> = Promise.resolve(); + + override async deliver(options: { + mail: Mail; + recipients?: string[]; + }): Promise<MailDeliverResult> { + console.info( + "The mail deliverer is sync. Wait for last delivering done...", + ); + await this.#last; + const result = super.deliver(options); + this.#last = result.then( + () => {}, + () => {}, + ); + return result; + } +} + +export class RecipientFromHeadersHook implements MailDeliverHook { + constructor(public mailDomain: string) {} + + callback(context: MailDeliverContext) { + if (context.recipients.size !== 0) { + console.warn( + "Recipients are already filled. Won't set them with ones in headers.", + ); + } else { + context.mail + .startSimpleParse() + .sections() + .headers() + .recipients({ + domain: this.mailDomain, + }) + .forEach((r) => context.recipients.add(r)); + + console.info( + "Recipients found from mail headers: " + + [...context.recipients].join(", "), + ); + } + return Promise.resolve(); + } +} + +export class FallbackRecipientHook implements MailDeliverHook { + constructor(public fallback: Set<string> = new Set()) {} + + callback(context: MailDeliverContext) { + if (context.recipients.size === 0) { + console.info( + "No recipients, fill with fallback: " + [...this.fallback].join(", "), + ); + this.fallback.forEach((a) => context.recipients.add(a)); + } + return Promise.resolve(); + } +} + +export class AliasRecipientMailHook implements MailDeliverHook { + #aliasFile; + + constructor(aliasFile: string) { + this.#aliasFile = aliasFile; + } + + async #parseAliasFile(): Promise<Map<string, string>> { + const result = new Map(); + if ((await Deno.stat(this.#aliasFile)).isFile) { + console.info(`Found recipients alias file: ${this.#aliasFile}.`); + const text = await Deno.readTextFile(this.#aliasFile); + const csv = parse(text); + for (const [real, ...aliases] of csv) { + aliases.forEach((a) => result.set(a, real)); + } + } + return result; + } + + async callback(context: MailDeliverContext) { + const aliases = await this.#parseAliasFile(); + for (const recipient of [...context.recipients]) { + const realRecipients = aliases.get(recipient); + if (realRecipients != null) { + console.info( + `Recipient alias resolved: ${recipient} => ${realRecipients}.`, + ); + context.recipients.delete(recipient); + context.recipients.add(realRecipients); + } + } + } +} |