From e18d101cae1dfcef29abd102d2908d429f4688d5 Mon Sep 17 00:00:00 2001 From: Yuqian Yang Date: Mon, 30 Jun 2025 14:25:02 +0800 Subject: mail: revert removing. --- deno/deno.json | 4 +- deno/mail-relay/app.ts | 83 ---------- deno/mail-relay/aws/app.ts | 315 ------------------------------------ deno/mail-relay/aws/deliver.ts | 63 -------- deno/mail-relay/aws/fetch.ts | 136 ---------------- deno/mail-relay/aws/mail.ts | 59 ------- deno/mail-relay/db.test.ts | 23 --- deno/mail-relay/db.ts | 146 ----------------- deno/mail-relay/deno.json | 18 --- deno/mail-relay/dovecot.ts | 213 ------------------------ deno/mail-relay/dumb-smtp-server.ts | 129 --------------- deno/mail-relay/mail-parsing.ts | 144 ----------------- deno/mail-relay/mail.test.ts | 121 -------------- deno/mail-relay/mail.ts | 234 --------------------------- deno/mail/app.ts | 83 ++++++++++ deno/mail/aws/app.ts | 315 ++++++++++++++++++++++++++++++++++++ deno/mail/aws/deliver.ts | 63 ++++++++ deno/mail/aws/fetch.ts | 136 ++++++++++++++++ deno/mail/aws/mail.ts | 59 +++++++ deno/mail/db.test.ts | 23 +++ deno/mail/db.ts | 146 +++++++++++++++++ deno/mail/deno.json | 18 +++ deno/mail/dovecot.ts | 213 ++++++++++++++++++++++++ deno/mail/dumb-smtp-server.ts | 129 +++++++++++++++ deno/mail/mail-parsing.ts | 144 +++++++++++++++++ deno/mail/mail.test.ts | 121 ++++++++++++++ deno/mail/mail.ts | 234 +++++++++++++++++++++++++++ 27 files changed, 1686 insertions(+), 1686 deletions(-) delete mode 100644 deno/mail-relay/app.ts delete mode 100644 deno/mail-relay/aws/app.ts delete mode 100644 deno/mail-relay/aws/deliver.ts delete mode 100644 deno/mail-relay/aws/fetch.ts delete mode 100644 deno/mail-relay/aws/mail.ts delete mode 100644 deno/mail-relay/db.test.ts delete mode 100644 deno/mail-relay/db.ts delete mode 100644 deno/mail-relay/deno.json delete mode 100644 deno/mail-relay/dovecot.ts delete mode 100644 deno/mail-relay/dumb-smtp-server.ts delete mode 100644 deno/mail-relay/mail-parsing.ts delete mode 100644 deno/mail-relay/mail.test.ts delete mode 100644 deno/mail-relay/mail.ts create mode 100644 deno/mail/app.ts create mode 100644 deno/mail/aws/app.ts create mode 100644 deno/mail/aws/deliver.ts create mode 100644 deno/mail/aws/fetch.ts create mode 100644 deno/mail/aws/mail.ts create mode 100644 deno/mail/db.test.ts create mode 100644 deno/mail/db.ts create mode 100644 deno/mail/deno.json create mode 100644 deno/mail/dovecot.ts create mode 100644 deno/mail/dumb-smtp-server.ts create mode 100644 deno/mail/mail-parsing.ts create mode 100644 deno/mail/mail.test.ts create mode 100644 deno/mail/mail.ts (limited to 'deno') diff --git a/deno/deno.json b/deno/deno.json index d4beef0..286451e 100644 --- a/deno/deno.json +++ b/deno/deno.json @@ -1,7 +1,7 @@ { - "workspace": ["./base", "./mail-relay", "./tools"], + "workspace": ["./base", "./mail", "./tools"], "tasks": { - "compile:mail-relay": "deno task --cwd=mail-relay compile" + "compile:mail": "deno task --cwd=mail compile" }, "imports": { "@std/collections": "jsr:@std/collections@^1.1.1", diff --git a/deno/mail-relay/app.ts b/deno/mail-relay/app.ts deleted file mode 100644 index 332c430..0000000 --- a/deno/mail-relay/app.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { Hono } from "hono"; -import { logger as honoLogger } from "hono/logger"; - -import { - AliasRecipientMailHook, - FallbackRecipientHook, - MailDeliverer, - RecipientFromHeadersHook, -} from "./mail.ts"; -import { DovecotMailDeliverer } from "./dovecot.ts"; -import { DumbSmtpServer } from "./dumb-smtp-server.ts"; - -export function createInbound( - { - fallback, - mailDomain, - aliasFile, - ldaPath, - doveadmPath, - }: { - fallback: string[]; - mailDomain: string; - aliasFile: string; - ldaPath: string; - doveadmPath: string; - }, -) { - const deliverer = new DovecotMailDeliverer(ldaPath, doveadmPath); - deliverer.preHooks.push( - new RecipientFromHeadersHook(mailDomain), - new FallbackRecipientHook(new Set(fallback)), - new AliasRecipientMailHook(aliasFile), - ); - return deliverer; -} - -export function createHono(outbound: MailDeliverer, inbound: MailDeliverer) { - const hono = new Hono(); - - hono.onError((err, c) => { - console.error("Hono handler threw an uncaught error.", err); - return c.json({ message: "Server error, check its log." }, 500); - }); - hono.use(honoLogger()); - hono.post("/send/raw", async (context) => { - const body = await context.req.text(); - if (body.trim().length === 0) { - return context.json({ message: "Can't send an empty mail." }, 400); - } else { - const result = await outbound.deliverRaw(body); - return context.json({ - awsMessageId: result.awsMessageId, - }); - } - }); - hono.post("/receive/raw", async (context) => { - await inbound.deliverRaw(await context.req.text()); - return context.json({ message: "Done!" }); - }); - - return hono; -} - -export function createSmtp(outbound: MailDeliverer) { - return new DumbSmtpServer(outbound); -} - -export async function sendMail(port: number) { - const decoder = new TextDecoder(); - let text = ""; - for await (const chunk of Deno.stdin.readable) { - text += decoder.decode(chunk); - } - - const res = await fetch(`http://127.0.0.1:${port}/send/raw`, { - method: "post", - body: text, - }); - const fn = res.ok ? "info" : "error"; - console[fn](res); - console[fn](await res.text()); - if (!res.ok) Deno.exit(-1); -} diff --git a/deno/mail-relay/aws/app.ts b/deno/mail-relay/aws/app.ts deleted file mode 100644 index e3f1114..0000000 --- a/deno/mail-relay/aws/app.ts +++ /dev/null @@ -1,315 +0,0 @@ -import { join } from "@std/path"; -import { z } from "zod"; -import { Hono } from "hono"; -import { zValidator } from "@hono/zod-validator"; -import { FetchHttpHandler } from "@smithy/fetch-http-handler"; -// @ts-types="npm:@types/yargs" -import yargs from "yargs"; - -import { ConfigDefinition, ConfigProvider } from "@crupest/base/config"; -import { CronTask } from "@crupest/base/cron"; - -import { DbService } from "../db.ts"; -import { createHono, createInbound, createSmtp, sendMail } from "../app.ts"; -import { DovecotMailDeliverer } from "../dovecot.ts"; -import { MailDeliverer } from "../mail.ts"; -import { - AwsMailMessageIdRewriteHook, - AwsMailMessageIdSaveHook, -} from "./mail.ts"; -import { AwsMailDeliverer } from "./deliver.ts"; -import { AwsMailFetcher, LiveMailNotFoundError } from "./fetch.ts"; - - -const PREFIX = "crupest-mail-server"; -const CONFIG_DEFINITIONS = { - dataPath: { - description: "Path to save app persistent data.", - default: ".", - }, - mailDomain: { - description: - "The part after `@` of an address. Used to determine local recipients.", - }, - httpHost: { - description: "Listening address for http server.", - default: "0.0.0.0", - }, - httpPort: { description: "Listening port for http server.", default: "2345" }, - smtpHost: { - description: "Listening address for dumb smtp server.", - default: "127.0.0.1", - }, - smtpPort: { - description: "Listening port for dumb smtp server.", - default: "2346", - }, - ldaPath: { - description: "full path of lda executable", - default: "/dovecot/libexec/dovecot/dovecot-lda", - }, - doveadmPath: { - description: "full path of doveadm executable", - default: "/dovecot/bin/doveadm", - }, - inboundFallback: { - description: "comma separated addresses used as fallback recipients", - default: "", - }, - awsInboundPath: { - description: "(random set) path for aws sns", - }, - awsInboundKey: { - description: "(random set) http header Authorization for aws sns", - }, - awsRegion: { - description: "aws region", - }, - awsUser: { - description: "aws access key id", - }, - awsPassword: { - description: "aws secret access key", - secret: true, - }, - awsMailBucket: { - description: "aws s3 bucket saving raw mails", - secret: true, - }, -} as const satisfies ConfigDefinition; - -function createAwsOptions({ - user, - password, - region, -}: { - user: string; - password: string; - region: string; -}) { - return { - credentials: () => - Promise.resolve({ - accessKeyId: user, - secretAccessKey: password, - }), - requestHandler: new FetchHttpHandler(), - region, - }; -} - -function createOutbound( - awsOptions: ReturnType, - db: DbService, - local?: DovecotMailDeliverer, -) { - const deliverer = new AwsMailDeliverer(awsOptions); - deliverer.preHooks.push( - new AwsMailMessageIdRewriteHook(db.messageIdToAws.bind(db)), - ); - deliverer.postHooks.push( - new AwsMailMessageIdSaveHook( - async (original, aws, context) => { - await db.addMessageIdMap({ message_id: original, aws_message_id: aws }); - void local?.saveNewSent(context.logTag, context.mail, original); - }, - ), - ); - return deliverer; -} - -function setupAwsHono( - hono: Hono, - options: { - path: string; - auth: string; - fetcher: AwsMailFetcher; - deliverer: MailDeliverer; - }, -) { - let counter = 1; - - hono.post( - `/${options.path}`, - async (ctx, next) => { - const auth = ctx.req.header("Authorization"); - if (auth !== options.auth) { - return ctx.json({ message: "Bad auth!" }, 403); - } - await next(); - }, - zValidator( - "json", - z.object({ - key: z.string(), - recipients: z.optional(z.array(z.string())), - }), - ), - async (ctx) => { - const { fetcher, deliverer } = options; - const { key, recipients } = ctx.req.valid("json"); - try { - await fetcher.deliverLiveMail( - `[inbound ${counter++}]`, - key, - deliverer, - recipients, - ); - } catch (e) { - if (e instanceof LiveMailNotFoundError) { - return ctx.json({ message: e.message }); - } - throw e; - } - return ctx.json({ message: "Done!" }); - }, - ); -} - -function createCron(fetcher: AwsMailFetcher, deliverer: MailDeliverer) { - return new CronTask({ - name: "live-mail-recycler", - interval: 6 * 3600 * 1000, - callback: () => { - return fetcher.recycleLiveMails(deliverer); - }, - startNow: true, - }); -} - -function createBaseServices() { - const config = new ConfigProvider(PREFIX, CONFIG_DEFINITIONS); - Deno.mkdirSync(config.get("dataPath"), { recursive: true }); - return { config }; -} - -function createAwsFetchOnlyServices() { - const services = createBaseServices(); - const { config } = services; - - const awsOptions = createAwsOptions({ - user: config.get("awsUser"), - password: config.get("awsPassword"), - region: config.get("awsRegion"), - }); - const fetcher = new AwsMailFetcher(awsOptions, config.get("awsMailBucket")); - - return { ...services, awsOptions, fetcher }; -} - -function createAwsRecycleOnlyServices() { - const services = createAwsFetchOnlyServices(); - const { config } = services; - - const inbound = createInbound({ - fallback: config.getList("inboundFallback"), - ldaPath: config.get("ldaPath"), - doveadmPath: config.get("doveadmPath"), - aliasFile: join(config.get("dataPath"), "aliases.csv"), - mailDomain: config.get("mailDomain"), - }); - - return { ...services, inbound }; -} - -function createAwsServices() { - const services = createAwsRecycleOnlyServices(); - const { config, awsOptions, inbound } = services; - - const dbService = new DbService(join(config.get("dataPath"), "db.sqlite")); - const outbound = createOutbound(awsOptions, dbService, inbound); - - return { ...services, dbService, outbound }; -} - -function createServerServices() { - const services = createAwsServices(); - const { config, outbound, inbound, fetcher } = services; - - const smtp = createSmtp(outbound); - const hono = createHono(outbound, inbound); - - setupAwsHono(hono, { - path: config.get("awsInboundPath"), - auth: config.get("awsInboundKey"), - fetcher, - deliverer: inbound, - }); - - return { ...services, smtp, hono }; -} - -function serve(cron: boolean = false) { - const { config, fetcher, inbound, smtp, hono } = createServerServices(); - smtp.serve({ - hostname: config.get("smtpHost"), - port: config.getInt("smtpPort"), - }); - Deno.serve( - { - hostname: config.get("httpHost"), - port: config.getInt("httpPort"), - }, - hono.fetch, - ); - - if (cron) { - createCron(fetcher, inbound); - } -} - -async function listLives() { - const { fetcher } = createAwsFetchOnlyServices(); - const liveMails = await fetcher.listLiveMails(); - console.info(`Total ${liveMails.length}:`); - if (liveMails.length !== 0) { - console.info(liveMails.join("\n")); - } -} - -async function recycleLives() { - const { fetcher, inbound } = createAwsRecycleOnlyServices(); - await fetcher.recycleLiveMails(inbound); -} - -if (import.meta.main) { - await yargs(Deno.args) - .scriptName("mail-relay") - .command({ - command: "sendmail", - describe: "send mail via this server's endpoint", - handler: async (_argv) => { - const { config } = createBaseServices(); - await sendMail(config.getInt("httpPort")); - }, - }) - .command({ - command: "live", - describe: "work with live mails", - builder: (builder) => { - return builder - .command({ - command: "list", - describe: "list live mails", - handler: listLives, - }) - .command({ - command: "recycle", - describe: "recycle all live mails", - handler: recycleLives, - }) - .demandCommand(1, "One command must be specified."); - }, - handler: () => {}, - }) - .command({ - command: "serve", - describe: "start the http and smtp servers", - builder: (builder) => builder.option("real", { type: "boolean" }), - handler: (argv) => serve(argv.real), - }) - .demandCommand(1, "One command must be specified.") - .help() - .strict() - .parse(); -} diff --git a/deno/mail-relay/aws/deliver.ts b/deno/mail-relay/aws/deliver.ts deleted file mode 100644 index 0195369..0000000 --- a/deno/mail-relay/aws/deliver.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { - SendEmailCommand, - SESv2Client, - SESv2ClientConfig, -} from "@aws-sdk/client-sesv2"; - -import { Mail, MailDeliverContext, MailDeliverer } from "../mail.ts"; - -declare module "../mail.ts" { - interface MailDeliverResult { - awsMessageId?: string; - } -} - -export class AwsMailDeliverer extends MailDeliverer { - readonly name = "aws"; - readonly #aws; - readonly #ses; - - constructor(aws: SESv2ClientConfig) { - super(true); - this.#aws = aws; - this.#ses = new SESv2Client(aws); - } - - protected override async doDeliver( - mail: Mail, - context: MailDeliverContext, - ): Promise { - try { - const sendCommand = new SendEmailCommand({ - Content: { - Raw: { Data: mail.toUtf8Bytes() }, - }, - }); - - console.info(context.logTag, "Calling aws send-email api..."); - const res = await this.#ses.send(sendCommand); - if (res.MessageId == null) { - console.warn( - context.logTag, - "AWS send-email returned null message id.", - ); - } else { - context.result.awsMessageId = - `${res.MessageId}@${this.#aws.region}.amazonses.com`; - } - - context.result.smtpMessage = - `AWS Message ID: ${context.result.awsMessageId}`; - context.result.recipients.set("*", { - kind: "success", - message: `Succeeded to call aws send-email api.`, - }); - } catch (cause) { - context.result.recipients.set("*", { - kind: "failure", - message: "A JS error was thrown when calling aws send-email." + cause, - cause, - }); - } - } -} diff --git a/deno/mail-relay/aws/fetch.ts b/deno/mail-relay/aws/fetch.ts deleted file mode 100644 index 2154972..0000000 --- a/deno/mail-relay/aws/fetch.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { - CopyObjectCommand, - DeleteObjectCommand, - GetObjectCommand, - ListObjectsV2Command, - NoSuchKey, - S3Client, - S3ClientConfig, -} from "@aws-sdk/client-s3"; - -import { DateUtils } from "@crupest/base"; - -import { Mail } from "../mail.ts"; -import { MailDeliverer } from "../mail.ts"; - -export class LiveMailNotFoundError extends Error {} - -async function s3MoveObject( - client: S3Client, - bucket: string, - path: string, - newPath: string, -): Promise { - const copyCommand = new CopyObjectCommand({ - Bucket: bucket, - Key: newPath, - CopySource: `${bucket}/${path}`, - }); - await client.send(copyCommand); - - const deleteCommand = new DeleteObjectCommand({ - Bucket: bucket, - Key: path, - }); - await client.send(deleteCommand); -} - -const AWS_SES_S3_SETUP_TAG = "AMAZON_SES_SETUP_NOTIFICATION"; - -export class AwsMailFetcher { - readonly #livePrefix = "mail/live/"; - readonly #archivePrefix = "mail/archive/"; - readonly #s3; - readonly #bucket; - - constructor(aws: S3ClientConfig, bucket: string) { - this.#s3 = new S3Client(aws); - this.#bucket = bucket; - } - - async listLiveMails(): Promise { - const listCommand = new ListObjectsV2Command({ - Bucket: this.#bucket, - Prefix: this.#livePrefix, - }); - const res = await this.#s3.send(listCommand); - - if (res.Contents == null) { - console.warn("S3 API returned null Content."); - return []; - } - - const result: string[] = []; - for (const object of res.Contents) { - if (object.Key == null) { - console.warn("S3 API returned null Key."); - continue; - } - - if (object.Key.endsWith(AWS_SES_S3_SETUP_TAG)) continue; - - result.push(object.Key.slice(this.#livePrefix.length)); - } - return result; - } - - async deliverLiveMail( - logTag: string, - s3Key: string, - deliverer: MailDeliverer, - recipients?: string[], - ) { - console.info(logTag, `Fetching live mail ${s3Key}...`); - const mailPath = `${this.#livePrefix}${s3Key}`; - const command = new GetObjectCommand({ - Bucket: this.#bucket, - Key: mailPath, - }); - - let rawMail; - - try { - const res = await this.#s3.send(command); - if (res.Body == null) { - throw new Error("S3 API returns a null body."); - } - rawMail = await res.Body.transformToString(); - } catch (cause) { - if (cause instanceof NoSuchKey) { - const message = - `Live mail ${s3Key} is not found. Perhaps already delivered?`; - console.error(message, cause); - throw new LiveMailNotFoundError(message); - } - throw cause; - } - - const mail = new Mail(rawMail); - await deliverer.deliver({ mail, recipients }); - - const { date } = new Mail(rawMail).parsed; - const dateString = date != null - ? DateUtils.toFileNameString(date, true) - : "invalid-date"; - const newPath = `${this.#archivePrefix}${dateString}/${s3Key}`; - - console.info(logTag, `Archiving live mail ${s3Key} to ${newPath}...`); - await s3MoveObject(this.#s3, this.#bucket, mailPath, newPath); - - console.info(logTag, `Done deliver live mail ${s3Key}.`); - } - - async recycleLiveMails(deliverer: MailDeliverer) { - console.info("Begin to recycle live mails..."); - const mails = await this.listLiveMails(); - console.info(`Found ${mails.length} live mails`); - let counter = 1; - for (const s3Key of mails) { - await this.deliverLiveMail( - `[${counter++}/${mails.length}]`, - s3Key, - deliverer, - ); - } - } -} diff --git a/deno/mail-relay/aws/mail.ts b/deno/mail-relay/aws/mail.ts deleted file mode 100644 index 26f3ea0..0000000 --- a/deno/mail-relay/aws/mail.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { MailDeliverContext, MailDeliverHook } from "../mail.ts"; - -export class AwsMailMessageIdRewriteHook implements MailDeliverHook { - readonly #lookup; - - constructor(lookup: (origin: string) => Promise) { - this.#lookup = lookup; - } - - async callback(context: MailDeliverContext): Promise { - const addresses = context.mail.simpleFindAllAddresses(); - for (const address of addresses) { - const awsMessageId = await this.#lookup(address); - if (awsMessageId != null && awsMessageId.length !== 0) { - console.info( - context.logTag, - `Rewrite address-line string in mail: ${address} => ${awsMessageId}.`, - ); - context.mail.raw = context.mail.raw.replaceAll(address, awsMessageId); - } - } - } -} - -export class AwsMailMessageIdSaveHook implements MailDeliverHook { - readonly #record; - - constructor( - record: ( - original: string, - aws: string, - context: MailDeliverContext, - ) => Promise, - ) { - this.#record = record; - } - - async callback(context: MailDeliverContext): Promise { - const { messageId } = context.mail.parsed; - if (messageId == null) { - console.warn( - context.logTag, - "Original mail doesn't have message id, skip saving message id map.", - ); - return; - } - if (context.result.awsMessageId != null) { - console.info( - context.logTag, - `Save message id map: ${messageId} => ${context.result.awsMessageId}.`, - ); - context.mail.raw = context.mail.raw.replaceAll( - messageId, - context.result.awsMessageId, - ); - await this.#record(messageId, context.result.awsMessageId, context); - } - } -} diff --git a/deno/mail-relay/db.test.ts b/deno/mail-relay/db.test.ts deleted file mode 100644 index 60035c4..0000000 --- a/deno/mail-relay/db.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { describe, it } from "@std/testing/bdd"; -import { expect } from "@std/expect/expect"; - -import { DbService } from "./db.ts"; - -describe("DbService", () => { - const mockRow = { - message_id: "mock-message-id@mock.mock", - aws_message_id: "mock-aws-message-id@mock.mock", - }; - - it("works", async () => { - const db = new DbService(":memory:"); - await db.migrate(); - await db.addMessageIdMap(mockRow); - expect(await db.messageIdToAws(mockRow.message_id)).toBe( - mockRow.aws_message_id, - ); - expect(await db.messageIdFromAws(mockRow.aws_message_id)).toBe( - mockRow.message_id, - ); - }); -}); diff --git a/deno/mail-relay/db.ts b/deno/mail-relay/db.ts deleted file mode 100644 index 062700b..0000000 --- a/deno/mail-relay/db.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { - Generated, - Insertable, - Kysely, - Migration, - Migrator, - SqliteDatabase, - SqliteDialect, - SqliteStatement, -} from "kysely"; -import * as sqlite from "@db/sqlite"; - -class SqliteStatementAdapter implements SqliteStatement { - constructor(public readonly stmt: sqlite.Statement) {} - - get reader(): boolean { - return this.stmt.columnNames().length >= 1; - } - - all(parameters: readonly unknown[]): unknown[] { - return this.stmt.all(...(parameters as sqlite.BindValue[])); - } - - iterate(parameters: readonly unknown[]): IterableIterator { - return this.stmt.iter(...(parameters as sqlite.BindValue[])); - } - - run(parameters: readonly unknown[]): { - changes: number | bigint; - lastInsertRowid: number | bigint; - } { - const { db } = this.stmt; - const totalChangesBefore = db.totalChanges; - const changes = this.stmt.run(...(parameters as sqlite.BindValue[])); - return { - changes: totalChangesBefore === db.totalChanges ? 0 : changes, - lastInsertRowid: db.lastInsertRowId, - }; - } -} - -class SqliteDatabaseAdapter implements SqliteDatabase { - constructor(public readonly db: sqlite.Database) {} - - prepare(sql: string): SqliteStatementAdapter { - return new SqliteStatementAdapter(this.db.prepare(sql)); - } - - close(): void { - this.db.close(); - } -} - -export class DbError extends Error {} - -interface AwsMessageIdMapTable { - id: Generated; - message_id: string; - aws_message_id: string; -} - -interface Database { - aws_message_id_map: AwsMessageIdMapTable; -} - -const migrations: Record = { - "0001-init": { - // deno-lint-ignore no-explicit-any - async up(db: Kysely): Promise { - await db.schema - .createTable("aws_message_id_map") - .addColumn("id", "integer", (col) => col.primaryKey().autoIncrement()) - .addColumn("message_id", "text", (col) => col.notNull().unique()) - .addColumn("aws_message_id", "text", (col) => col.notNull().unique()) - .execute(); - - for (const column of ["message_id", "aws_message_id"]) { - await db.schema - .createIndex(`aws_message_id_map_${column}`) - .on("aws_message_id_map") - .column(column) - .execute(); - } - }, - - // deno-lint-ignore no-explicit-any - async down(db: Kysely): Promise { - await db.schema.dropTable("aws_message_id_map").execute(); - }, - }, -}; - -export class DbService { - #db; - #kysely; - #migrator; - - constructor(public readonly path: string) { - this.#db = new sqlite.Database(path); - this.#kysely = new Kysely({ - dialect: new SqliteDialect({ - database: new SqliteDatabaseAdapter(this.#db), - }), - }); - this.#migrator = new Migrator({ - db: this.#kysely, - provider: { - getMigrations(): Promise> { - return Promise.resolve(migrations); - }, - }, - }); - } - - async migrate(): Promise { - await this.#migrator.migrateToLatest(); - } - - async addMessageIdMap( - mail: Insertable, - ): Promise { - const inserted = await this.#kysely - .insertInto("aws_message_id_map") - .values(mail) - .executeTakeFirstOrThrow(); - return Number(inserted.insertId!); - } - - async messageIdToAws(messageId: string): Promise { - const row = await this.#kysely - .selectFrom("aws_message_id_map") - .where("message_id", "=", messageId) - .select("aws_message_id") - .executeTakeFirst(); - return row?.aws_message_id ?? null; - } - - async messageIdFromAws(awsMessageId: string): Promise { - const row = await this.#kysely - .selectFrom("aws_message_id_map") - .where("aws_message_id", "=", awsMessageId) - .select("message_id") - .executeTakeFirst(); - return row?.message_id ?? null; - } -} diff --git a/deno/mail-relay/deno.json b/deno/mail-relay/deno.json deleted file mode 100644 index 9105747..0000000 --- a/deno/mail-relay/deno.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "version": "0.1.0", - "tasks": { - "run": "deno run -A aws/app.ts", - "compile": "deno compile -o out/crupest-relay -A aws/app.ts" - }, - "imports": { - "@aws-sdk/client-s3": "npm:@aws-sdk/client-s3@^3.821.0", - "@aws-sdk/client-sesv2": "npm:@aws-sdk/client-sesv2@^3.821.0", - "@db/sqlite": "jsr:@db/sqlite@^0.12.0", - "@hono/zod-validator": "npm:@hono/zod-validator@^0.7.0", - "@smithy/fetch-http-handler": "npm:@smithy/fetch-http-handler@^5.0.4", - "email-addresses": "npm:email-addresses@^5.0.0", - "hono": "npm:hono@^4.7.11", - "kysely": "npm:kysely@^0.28.2", - "zod": "npm:zod@^3.25.48" - } -} diff --git a/deno/mail-relay/dovecot.ts b/deno/mail-relay/dovecot.ts deleted file mode 100644 index 4fe2f41..0000000 --- a/deno/mail-relay/dovecot.ts +++ /dev/null @@ -1,213 +0,0 @@ -import { Mail, MailDeliverContext, MailDeliverer } from "./mail.ts"; - -// https://doc.dovecot.org/main/core/man/dovecot-lda.1.html -const ldaExitCodeMessageMap = new Map(); -ldaExitCodeMessageMap.set(67, "recipient user not known"); -ldaExitCodeMessageMap.set(75, "temporary error"); - -type CommandResult = { - kind: "exit"; - status: Deno.CommandStatus; - logMessage: string; -} | { kind: "throw"; cause: unknown; logMessage: string }; - -async function runCommand( - bin: string, - options: { - logTag: string; - args: string[]; - stdin?: Uint8Array; - suppressResultLog?: boolean; - errorCodeMessageMap?: Map; - }, -): Promise { - const { logTag, args, stdin, suppressResultLog, errorCodeMessageMap } = - options; - - console.info(logTag, `Run external command ${bin} ${args.join(" ")}`); - - try { - // Create and spawn process. - const command = new Deno.Command(bin, { - args, - stdin: stdin == null ? "null" : "piped", - }); - const process = command.spawn(); - - // Write stdin if any. - if (stdin != null) { - const writer = process.stdin.getWriter(); - await writer.write(stdin); - writer.close(); - } - - // Wait for process to exit. - const status = await process.status; - - // Build log message string. - let message = `External command exited with code ${status.code}`; - if (status.signal != null) message += ` (signal: ${status.signal})`; - if (errorCodeMessageMap != null && errorCodeMessageMap.has(status.code)) { - message += `, ${errorCodeMessageMap.get(status.code)}`; - } - message += "."; - if (suppressResultLog !== true) console.log(logTag, message); - - // Return result. - return { - kind: "exit", - status, - logMessage: message, - }; - } catch (cause) { - const message = `A JS error was thrown when invoking external command:`; - if (suppressResultLog !== true) console.log(logTag, message); - return { kind: "throw", cause, logMessage: message + " " + cause }; - } -} - -export class DovecotMailDeliverer extends MailDeliverer { - readonly name = "dovecot"; - readonly #ldaPath; - readonly #doveadmPath; - - constructor( - ldaPath: string, - doveadmPath: string, - ) { - super(false); - this.#ldaPath = ldaPath; - this.#doveadmPath = doveadmPath; - } - - protected override async doDeliver( - mail: Mail, - context: MailDeliverContext, - ): Promise { - const utf8Bytes = mail.toUtf8Bytes(); - - const recipients = [...context.recipients]; - - if (recipients.length === 0) { - throw new Error( - "Failed to deliver to dovecot, no recipients are specified.", - ); - } - - for (const recipient of recipients) { - const result = await runCommand( - this.#ldaPath, - { - logTag: context.logTag, - args: ["-d", recipient], - stdin: utf8Bytes, - suppressResultLog: true, - errorCodeMessageMap: ldaExitCodeMessageMap, - }, - ); - - if (result.kind === "exit" && result.status.success) { - context.result.recipients.set(recipient, { - kind: "success", - message: result.logMessage, - }); - } else { - context.result.recipients.set(recipient, { - kind: "failure", - message: result.logMessage, - }); - } - } - } - - #queryArgs(mailbox: string, messageId: string) { - return ["mailbox", mailbox, "header", "Message-ID", `<${messageId}>`]; - } - - async #deleteMail( - logTag: string, - user: string, - mailbox: string, - messageId: string, - ): Promise { - await runCommand(this.#doveadmPath, { - logTag, - args: ["expunge", "-u", user, ...this.#queryArgs(mailbox, messageId)], - }); - } - - async #saveMail( - logTag: string, - user: string, - mailbox: string, - mail: Uint8Array, - ) { - await runCommand(this.#doveadmPath, { - logTag, - args: ["save", "-u", user, "-m", mailbox], - stdin: mail, - }); - } - - async #markAsRead( - logTag: string, - user: string, - mailbox: string, - messageId: string, - ) { - await runCommand(this.#doveadmPath, { - logTag, - args: [ - "flags", - "add", - "-u", - user, - "\\Seen", - ...this.#queryArgs(mailbox, messageId), - ], - }); - } - - async saveNewSent(logTag: string, mail: Mail, messageIdToDelete: string) { - console.info(logTag, "Save sent mail and delete ones with old message id."); - - // Try to get from and recipients from headers. - const { messageId, from, recipients } = mail.parsed; - - if (from == null) { - console.warn( - logTag, - "Failed to get sender (from) in headers, skip saving.", - ); - return; - } - - if (recipients.includes(from)) { - // So the mail should lie in the Inbox. - console.info( - logTag, - "One recipient of the mail is the sender itself, skip saving.", - ); - return; - } - - await this.#saveMail(logTag, from, "Sent", mail.toUtf8Bytes()); - if (messageId != null) { - await this.#markAsRead(logTag, from, "Sent", messageId); - } else { - console.warn( - "Message id of the mail is not found, skip marking as read.", - ); - } - - console.info( - logTag, - "Schedule deletion of old mails at 15,30,60 seconds later.", - ); - [15, 30, 60].forEach((seconds) => - setTimeout(() => { - void this.#deleteMail(logTag, from, "Sent", messageIdToDelete); - }, 1000 * seconds) - ); - } -} diff --git a/deno/mail-relay/dumb-smtp-server.ts b/deno/mail-relay/dumb-smtp-server.ts deleted file mode 100644 index 70d5ec0..0000000 --- a/deno/mail-relay/dumb-smtp-server.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { MailDeliverer } from "./mail.ts"; - -const CRLF = "\r\n"; - -function createResponses(host: string, port: number | string) { - const serverName = `[${host}]:${port}`; - return { - serverName, - READY: `220 ${serverName} SMTP Ready`, - EHLO: `250 ${serverName}`, - MAIL: "250 2.1.0 Sender OK", - RCPT: "250 2.1.5 Recipient OK", - DATA: "354 Start mail input; end with .", - QUIT: `211 2.0.0 ${serverName} closing connection`, - ACTIVE_CLOSE: "421 4.7.0 Please open a new connection to send more emails", - INVALID: "500 5.5.1 Error: command not recognized", - } as const; -} - -export class DumbSmtpServer { - #deliverer; - - constructor(deliverer: MailDeliverer) { - this.#deliverer = deliverer; - } - - async #handleConnection( - logTag: string, - conn: Deno.Conn, - responses: ReturnType, - ) { - using disposeStack = new DisposableStack(); - disposeStack.defer(() => { - console.info(logTag, "Close tcp connection."); - conn.close(); - }); - - console.info(logTag, "New tcp connection established."); - - const writer = conn.writable.getWriter(); - disposeStack.defer(() => writer.releaseLock()); - const reader = conn.readable.getReader(); - disposeStack.defer(() => reader.releaseLock()); - - const [decoder, encoder] = [new TextDecoder(), new TextEncoder()]; - const decode = (data: Uint8Array) => decoder.decode(data); - const send = async (s: string) => { - console.info(logTag, "Send line:", s); - await writer.write(encoder.encode(s + CRLF)); - }; - - let buffer: string = ""; - let rawMail: string | null = null; - - await send(responses["READY"]); - - while (true) { - const { value, done } = await reader.read(); - if (done) break; - - buffer += decode(value); - - while (true) { - const eolPos = buffer.indexOf(CRLF); - if (eolPos === -1) break; - - const line = buffer.slice(0, eolPos); - buffer = buffer.slice(eolPos + CRLF.length); - - if (rawMail == null) { - console.info(logTag, "Received line:", line); - const upperLine = line.toUpperCase(); - if (upperLine.startsWith("EHLO") || upperLine.startsWith("HELO")) { - await send(responses["EHLO"]); - } else if (upperLine.startsWith("MAIL FROM:")) { - await send(responses["MAIL"]); - } else if (upperLine.startsWith("RCPT TO:")) { - await send(responses["RCPT"]); - } else if (upperLine === "DATA") { - await send(responses["DATA"]); - console.info(logTag, "Begin to receive mail data..."); - rawMail = ""; - } else if (upperLine === "QUIT") { - await send(responses["QUIT"]); - return; - } else { - await send(responses["INVALID"]); - return; - } - } else { - if (line === ".") { - try { - console.info(logTag, "Mail data received, begin to relay..."); - const { smtpMessage } = await this.#deliverer.deliverRaw(rawMail); - await send(`250 2.6.0 ${smtpMessage}`); - rawMail = null; - } catch (err) { - console.error(logTag, "Relay failed.", err); - await send("554 5.3.0 Error: check server log"); - } - await send(responses["ACTIVE_CLOSE"]); - } else { - const dataLine = line.startsWith("..") ? line.slice(1) : line; - rawMail += dataLine + CRLF; - } - } - } - } - } - - async serve(options: { hostname: string; port: number }) { - const listener = Deno.listen(options); - const responses = createResponses(options.hostname, options.port); - console.info( - `Dumb SMTP server starts to listen on ${responses.serverName}.`, - ); - - let counter = 1; - - for await (const conn of listener) { - const logTag = `[outbound ${counter++}]`; - try { - await this.#handleConnection(logTag, conn, responses); - } catch (cause) { - console.error(logTag, "A JS error was thrown by handler:", cause); - } - } - } -} diff --git a/deno/mail-relay/mail-parsing.ts b/deno/mail-relay/mail-parsing.ts deleted file mode 100644 index 8e9697d..0000000 --- a/deno/mail-relay/mail-parsing.ts +++ /dev/null @@ -1,144 +0,0 @@ -import emailAddresses from "email-addresses"; - -class MailParsingError extends Error {} - -function parseHeaderSection(section: string) { - const headers = [] as [key: string, value: string][]; - - let field: string | null = null; - let lineNumber = 1; - - const handleField = () => { - if (field == null) return; - const sepPos = field.indexOf(":"); - if (sepPos === -1) { - throw new MailParsingError( - `Expect ':' in the header field line: ${field}`, - ); - } - headers.push([field.slice(0, sepPos).trim(), field.slice(sepPos + 1)]); - field = null; - }; - - for (const line of section.trimEnd().split(/\r?\n|\r/)) { - if (line.match(/^\s/)) { - if (field == null) { - throw new MailParsingError("Header section starts with a space."); - } - field += line; - } else { - handleField(); - field = line; - } - lineNumber += 1; - } - - handleField(); - - return headers; -} - -function findFirst(fields: readonly [string, string][], key: string) { - for (const [k, v] of fields) { - if (key.toLowerCase() === k.toLowerCase()) return v; - } - return undefined; -} - -function findMessageId(fields: readonly [string, string][]) { - const messageIdField = findFirst(fields, "message-id"); - if (messageIdField == null) return undefined; - - const match = messageIdField.match(/\<(.*?)\>/); - if (match != null) { - return match[1]; - } else { - console.warn(`Invalid syntax in header 'message-id': ${messageIdField}`); - return undefined; - } -} - -function findDate(fields: readonly [string, string][]) { - const dateField = findFirst(fields, "date"); - if (dateField == null) return undefined; - - const date = new Date(dateField); - if (isNaN(date.getTime())) { - console.warn(`Invalid date string in header 'date': ${dateField}`); - return undefined; - } - return date; -} - -function findFrom(fields: readonly [string, string][]) { - const fromField = findFirst(fields, "from"); - if (fromField == null) return undefined; - - const addr = emailAddresses.parseOneAddress(fromField); - return addr?.type === "mailbox" ? addr.address : undefined; -} - -function findRecipients(fields: readonly [string, string][]) { - const headers = ["to", "cc", "bcc", "x-original-to"]; - const recipients = new Set(); - for (const [key, value] of fields) { - if (headers.includes(key.toLowerCase())) { - emailAddresses - .parseAddressList(value) - ?.flatMap((a) => (a.type === "mailbox" ? a : a.addresses)) - ?.forEach(({ address }) => recipients.add(address)); - } - } - return recipients; -} - -function parseSections(raw: string) { - const twoEolMatch = raw.match(/(\r?\n)(\r?\n)/); - if (twoEolMatch == null) { - throw new MailParsingError( - "No header/body section separator (2 successive EOLs) found.", - ); - } - - const [eol, sep] = [twoEolMatch[1], twoEolMatch[2]]; - - if (eol !== sep) { - console.warn("Different EOLs (\\r\\n, \\n) found."); - } - - return { - header: raw.slice(0, twoEolMatch.index!), - body: raw.slice(twoEolMatch.index! + eol.length + sep.length), - eol, - sep, - }; -} - -export type ParsedMail = Readonly<{ - header: string; - body: string; - sep: string; - eol: string; - headers: readonly [string, string][]; - messageId: string | undefined; - date: Date | undefined; - from: string | undefined; - recipients: readonly string[]; -}>; - -export function simpleParseMail(raw: string): ParsedMail { - const sections = Object.freeze(parseSections(raw)); - const headers = Object.freeze(parseHeaderSection(sections.header)); - const messageId = findMessageId(headers); - const date = findDate(headers); - const from = findFrom(headers); - const recipients = Object.freeze([...findRecipients(headers)]); - return Object.freeze({ - ...sections, - headers, - messageId, - date, - from, - recipients, - }); -} diff --git a/deno/mail-relay/mail.test.ts b/deno/mail-relay/mail.test.ts deleted file mode 100644 index a8204be..0000000 --- a/deno/mail-relay/mail.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { describe, it } from "@std/testing/bdd"; -import { expect, fn } from "@std/expect"; - -import { Mail, MailDeliverContext, MailDeliverer } from "./mail.ts"; - -const mockDate = "Fri, 02 May 2025 08:33:02 +0000"; -const mockMessageId = "mock-message-id@from.mock"; -const mockMessageId2 = "mock-message-id-2@from.mock"; -const mockFromAddress = "mock@from.mock"; -const mockCcAddress = "mock@cc.mock"; -const mockBodyStr = `This is body content. -Line 2 ${mockMessageId2} - -Line 4`; -const mockHeaders = [ - ["Content-Disposition", "inline"], - ["Content-Transfer-Encoding", "quoted-printable"], - ["MIME-Version", "1.0"], - ["X-Mailer", "MIME-tools 5.509 (Entity 5.509)"], - ["Content-Type", "text/plain; charset=utf-8"], - ["From", `"Mock From" <${mockFromAddress}>`], - [ - "To", - `"John \\"Big\\" Doe" , "Alice (Work)" , - undisclosed-recipients:;, "Group: Team" , - "Escaped, Name" , just@email.com, - "Comment (This is valid)" , - "Odd @Chars" , - "Non-ASCII 用户" , - admin@[192.168.1.1]`, - ], - ["CC", `Mock CC <${mockCcAddress}>`], - ["Subject", "A very long mock\n subject"], - ["Message-ID", `<${mockMessageId}>`], - ["Date", mockDate], -]; -const mockHeaderStr = mockHeaders.map((h) => h[0] + ": " + h[1]).join("\n"); -const mockMailStr = mockHeaderStr + "\n\n" + mockBodyStr; -const mockCrlfMailStr = mockMailStr.replaceAll("\n", "\r\n"); -const mockToAddresses = [ - "john@example.com", - "alice+work@example.com", - "team@company.com", - "escape@test.com", - "just@email.com", - "comment@domain.net", - "weird!#$%'*+-/=?^_`{|}~@char-test.com", - "user@例子.中国", - "admin@[192.168.1.1]", -]; - -describe("Mail", () => { - it("simple parse", () => { - const { parsed } = new Mail(mockMailStr); - expect(parsed.header).toEqual(mockHeaderStr); - expect(parsed.body).toEqual(mockBodyStr); - expect(parsed.sep).toBe("\n"); - expect(parsed.eol).toBe("\n"); - }); - - it("simple parse crlf", () => { - const { parsed } = new Mail(mockCrlfMailStr); - expect(parsed.sep).toBe("\r\n"); - expect(parsed.eol).toBe("\r\n"); - }); - - it("simple parse date", () => { - expect( - new Mail(mockMailStr).parsed.date, - ).toEqual(new Date(mockDate)); - }); - - it("simple parse headers", () => { - expect( - new Mail(mockMailStr).parsed.headers, - ).toEqual(mockHeaders.map((h) => [h[0], " " + h[1].replaceAll("\n", "")])); - }); - - it("parse recipients", () => { - const mail = new Mail(mockMailStr); - expect([...mail.parsed.recipients]).toEqual([ - ...mockToAddresses, - mockCcAddress, - ]); - }); - - it("find all addresses", () => { - const mail = new Mail(mockMailStr); - expect(mail.simpleFindAllAddresses()).toEqual([ - "mock@from.mock", - "john@example.com", - "alice+work@example.com", - "team@company.com", - "escape@test.com", - "just@email.com", - "comment@domain.net", - "mock@cc.mock", - "mock-message-id@from.mock", - "mock-message-id-2@from.mock", - ]); - }); -}); - -describe("MailDeliverer", () => { - class MockMailDeliverer extends MailDeliverer { - name = "mock"; - override doDeliver = fn((_: Mail, ctx: MailDeliverContext) => { - ctx.result.recipients.set("*", { - kind: "success", - message: "success message", - }); - return Promise.resolve(); - }) as MailDeliverer["doDeliver"]; - } - const mockDeliverer = new MockMailDeliverer(false); - - it("deliver success", async () => { - await mockDeliverer.deliverRaw(mockMailStr); - expect(mockDeliverer.doDeliver).toHaveBeenCalledTimes(1); - }); -}); diff --git a/deno/mail-relay/mail.ts b/deno/mail-relay/mail.ts deleted file mode 100644 index 9cc591c..0000000 --- a/deno/mail-relay/mail.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { encodeBase64 } from "@std/encoding/base64"; -import { parse } from "@std/csv/parse"; -import { simpleParseMail } from "./mail-parsing.ts"; - -export class Mail { - #raw; - #parsed; - - constructor(raw: string) { - this.#raw = raw; - this.#parsed = simpleParseMail(raw); - } - - get raw() { - return this.#raw; - } - - set raw(value) { - this.#raw = value; - this.#parsed = simpleParseMail(value); - } - - get parsed() { - return this.#parsed; - } - - toUtf8Bytes(): Uint8Array { - const utf8Encoder = new TextEncoder(); - return utf8Encoder.encode(this.raw); - } - - toBase64(): string { - return encodeBase64(this.raw); - } - - simpleFindAllAddresses(): string[] { - const re = /,?\?,?/gi; - return [...this.raw.matchAll(re)].map((m) => m[1]); - } -} - -export interface MailDeliverRecipientResult { - kind: "success" | "failure"; - message?: string; - cause?: unknown; -} - -export class MailDeliverResult { - message?: string; - smtpMessage?: string; - recipients = new Map(); - constructor(public mail: Mail) {} - - get hasFailure() { - return this.recipients.values().some((v) => v.kind !== "success"); - } - - generateLogMessage(prefix: string) { - const lines = []; - if (this.message != null) lines.push(`${prefix} message: ${this.message}`); - if (this.smtpMessage != null) { - lines.push(`${prefix} smtpMessage: ${this.smtpMessage}`); - } - for (const [name, result] of this.recipients.entries()) { - const { kind, message } = result; - lines.push(`${prefix} (${name}): ${kind} ${message}`); - } - return lines.join("\n"); - } -} - -export class MailDeliverContext { - readonly recipients: Set = new Set(); - readonly result; - - constructor(public logTag: string, public mail: Mail) { - this.result = new MailDeliverResult(this.mail); - } -} - -export interface MailDeliverHook { - callback(context: MailDeliverContext): Promise; -} - -export abstract class MailDeliverer { - #counter = 1; - #last?: Promise; - - abstract name: string; - preHooks: MailDeliverHook[] = []; - postHooks: MailDeliverHook[] = []; - - constructor(public sync: boolean) {} - - protected abstract doDeliver( - mail: Mail, - context: MailDeliverContext, - ): Promise; - - async deliverRaw(rawMail: string) { - return await this.deliver({ mail: new Mail(rawMail) }); - } - - async #deliverCore(context: MailDeliverContext) { - for (const hook of this.preHooks) { - await hook.callback(context); - } - - await this.doDeliver(context.mail, context); - - for (const hook of this.postHooks) { - await hook.callback(context); - } - } - - async deliver(options: { - mail: Mail; - recipients?: string[]; - logTag?: string; - }): Promise { - const logTag = options.logTag ?? `[${this.name} ${this.#counter}]`; - this.#counter++; - - if (this.#last != null) { - console.info(logTag, "Wait for last delivering done..."); - await this.#last; - } - - const context = new MailDeliverContext( - logTag, - options.mail, - ); - options.recipients?.forEach((r) => context.recipients.add(r)); - - console.info(context.logTag, "Begin to deliver mail..."); - - const deliverPromise = this.#deliverCore(context); - - if (this.sync) { - this.#last = deliverPromise.then(() => {}, () => {}); - } - - await deliverPromise; - this.#last = undefined; - - console.info(context.logTag, "Deliver result:"); - console.info(context.result.generateLogMessage(context.logTag)); - - if (context.result.hasFailure) { - throw new Error("Failed to deliver to some recipients."); - } - - return context.result; - } -} - -export class RecipientFromHeadersHook implements MailDeliverHook { - constructor(public mailDomain: string) {} - - callback(context: MailDeliverContext) { - if (context.recipients.size !== 0) { - console.warn( - context.logTag, - "Recipients are already filled, skip inferring from headers.", - ); - } else { - [...context.mail.parsed.recipients].filter((r) => - r.endsWith("@" + this.mailDomain) - ).forEach((r) => context.recipients.add(r)); - - console.info( - context.logTag, - "Use recipients inferred from mail headers:", - [...context.recipients].join(", "), - ); - } - return Promise.resolve(); - } -} - -export class FallbackRecipientHook implements MailDeliverHook { - constructor(public fallback: Set = new Set()) {} - - callback(context: MailDeliverContext) { - if (context.recipients.size === 0) { - console.info( - context.logTag, - "Use fallback recipients:" + [...this.fallback].join(", "), - ); - this.fallback.forEach((a) => context.recipients.add(a)); - } - return Promise.resolve(); - } -} - -export class AliasRecipientMailHook implements MailDeliverHook { - #aliasFile; - - constructor(aliasFile: string) { - this.#aliasFile = aliasFile; - } - - async #parseAliasFile(logTag: string): Promise> { - const result = new Map(); - if ((await Deno.stat(this.#aliasFile)).isFile) { - const text = await Deno.readTextFile(this.#aliasFile); - const csv = parse(text); - for (const [real, ...aliases] of csv) { - aliases.forEach((a) => result.set(a, real)); - } - } else { - console.warn( - logTag, - `Recipient alias file ${this.#aliasFile} is not found.`, - ); - } - return result; - } - - async callback(context: MailDeliverContext) { - const aliases = await this.#parseAliasFile(context.logTag); - for (const recipient of [...context.recipients]) { - const realRecipients = aliases.get(recipient); - if (realRecipients != null) { - console.info( - context.logTag, - `Recipient alias resolved: ${recipient} => ${realRecipients}.`, - ); - context.recipients.delete(recipient); - context.recipients.add(realRecipients); - } - } - } -} diff --git a/deno/mail/app.ts b/deno/mail/app.ts new file mode 100644 index 0000000..332c430 --- /dev/null +++ b/deno/mail/app.ts @@ -0,0 +1,83 @@ +import { Hono } from "hono"; +import { logger as honoLogger } from "hono/logger"; + +import { + AliasRecipientMailHook, + FallbackRecipientHook, + MailDeliverer, + RecipientFromHeadersHook, +} from "./mail.ts"; +import { DovecotMailDeliverer } from "./dovecot.ts"; +import { DumbSmtpServer } from "./dumb-smtp-server.ts"; + +export function createInbound( + { + fallback, + mailDomain, + aliasFile, + ldaPath, + doveadmPath, + }: { + fallback: string[]; + mailDomain: string; + aliasFile: string; + ldaPath: string; + doveadmPath: string; + }, +) { + const deliverer = new DovecotMailDeliverer(ldaPath, doveadmPath); + deliverer.preHooks.push( + new RecipientFromHeadersHook(mailDomain), + new FallbackRecipientHook(new Set(fallback)), + new AliasRecipientMailHook(aliasFile), + ); + return deliverer; +} + +export function createHono(outbound: MailDeliverer, inbound: MailDeliverer) { + const hono = new Hono(); + + hono.onError((err, c) => { + console.error("Hono handler threw an uncaught error.", err); + return c.json({ message: "Server error, check its log." }, 500); + }); + hono.use(honoLogger()); + hono.post("/send/raw", async (context) => { + const body = await context.req.text(); + if (body.trim().length === 0) { + return context.json({ message: "Can't send an empty mail." }, 400); + } else { + const result = await outbound.deliverRaw(body); + return context.json({ + awsMessageId: result.awsMessageId, + }); + } + }); + hono.post("/receive/raw", async (context) => { + await inbound.deliverRaw(await context.req.text()); + return context.json({ message: "Done!" }); + }); + + return hono; +} + +export function createSmtp(outbound: MailDeliverer) { + return new DumbSmtpServer(outbound); +} + +export async function sendMail(port: number) { + const decoder = new TextDecoder(); + let text = ""; + for await (const chunk of Deno.stdin.readable) { + text += decoder.decode(chunk); + } + + const res = await fetch(`http://127.0.0.1:${port}/send/raw`, { + method: "post", + body: text, + }); + const fn = res.ok ? "info" : "error"; + console[fn](res); + console[fn](await res.text()); + if (!res.ok) Deno.exit(-1); +} diff --git a/deno/mail/aws/app.ts b/deno/mail/aws/app.ts new file mode 100644 index 0000000..3c8305d --- /dev/null +++ b/deno/mail/aws/app.ts @@ -0,0 +1,315 @@ +import { join } from "@std/path"; +import { z } from "zod"; +import { Hono } from "hono"; +import { zValidator } from "@hono/zod-validator"; +import { FetchHttpHandler } from "@smithy/fetch-http-handler"; +// @ts-types="npm:@types/yargs" +import yargs from "yargs"; + +import { ConfigDefinition, ConfigProvider } from "@crupest/base/config"; +import { CronTask } from "@crupest/base/cron"; + +import { DbService } from "../db.ts"; +import { createHono, createInbound, createSmtp, sendMail } from "../app.ts"; +import { DovecotMailDeliverer } from "../dovecot.ts"; +import { MailDeliverer } from "../mail.ts"; +import { + AwsMailMessageIdRewriteHook, + AwsMailMessageIdSaveHook, +} from "./mail.ts"; +import { AwsMailDeliverer } from "./deliver.ts"; +import { AwsMailFetcher, LiveMailNotFoundError } from "./fetch.ts"; + + +const PREFIX = "crupest-mail-server"; +const CONFIG_DEFINITIONS = { + dataPath: { + description: "Path to save app persistent data.", + default: ".", + }, + mailDomain: { + description: + "The part after `@` of an address. Used to determine local recipients.", + }, + httpHost: { + description: "Listening address for http server.", + default: "0.0.0.0", + }, + httpPort: { description: "Listening port for http server.", default: "2345" }, + smtpHost: { + description: "Listening address for dumb smtp server.", + default: "127.0.0.1", + }, + smtpPort: { + description: "Listening port for dumb smtp server.", + default: "2346", + }, + ldaPath: { + description: "full path of lda executable", + default: "/dovecot/libexec/dovecot/dovecot-lda", + }, + doveadmPath: { + description: "full path of doveadm executable", + default: "/dovecot/bin/doveadm", + }, + inboundFallback: { + description: "comma separated addresses used as fallback recipients", + default: "", + }, + awsInboundPath: { + description: "(random set) path for aws sns", + }, + awsInboundKey: { + description: "(random set) http header Authorization for aws sns", + }, + awsRegion: { + description: "aws region", + }, + awsUser: { + description: "aws access key id", + }, + awsPassword: { + description: "aws secret access key", + secret: true, + }, + awsMailBucket: { + description: "aws s3 bucket saving raw mails", + secret: true, + }, +} as const satisfies ConfigDefinition; + +function createAwsOptions({ + user, + password, + region, +}: { + user: string; + password: string; + region: string; +}) { + return { + credentials: () => + Promise.resolve({ + accessKeyId: user, + secretAccessKey: password, + }), + requestHandler: new FetchHttpHandler(), + region, + }; +} + +function createOutbound( + awsOptions: ReturnType, + db: DbService, + local?: DovecotMailDeliverer, +) { + const deliverer = new AwsMailDeliverer(awsOptions); + deliverer.preHooks.push( + new AwsMailMessageIdRewriteHook(db.messageIdToAws.bind(db)), + ); + deliverer.postHooks.push( + new AwsMailMessageIdSaveHook( + async (original, aws, context) => { + await db.addMessageIdMap({ message_id: original, aws_message_id: aws }); + void local?.saveNewSent(context.logTag, context.mail, original); + }, + ), + ); + return deliverer; +} + +function setupAwsHono( + hono: Hono, + options: { + path: string; + auth: string; + fetcher: AwsMailFetcher; + deliverer: MailDeliverer; + }, +) { + let counter = 1; + + hono.post( + `/${options.path}`, + async (ctx, next) => { + const auth = ctx.req.header("Authorization"); + if (auth !== options.auth) { + return ctx.json({ message: "Bad auth!" }, 403); + } + await next(); + }, + zValidator( + "json", + z.object({ + key: z.string(), + recipients: z.optional(z.array(z.string())), + }), + ), + async (ctx) => { + const { fetcher, deliverer } = options; + const { key, recipients } = ctx.req.valid("json"); + try { + await fetcher.deliverLiveMail( + `[inbound ${counter++}]`, + key, + deliverer, + recipients, + ); + } catch (e) { + if (e instanceof LiveMailNotFoundError) { + return ctx.json({ message: e.message }); + } + throw e; + } + return ctx.json({ message: "Done!" }); + }, + ); +} + +function createCron(fetcher: AwsMailFetcher, deliverer: MailDeliverer) { + return new CronTask({ + name: "live-mail-recycler", + interval: 6 * 3600 * 1000, + callback: () => { + return fetcher.recycleLiveMails(deliverer); + }, + startNow: true, + }); +} + +function createBaseServices() { + const config = new ConfigProvider(PREFIX, CONFIG_DEFINITIONS); + Deno.mkdirSync(config.get("dataPath"), { recursive: true }); + return { config }; +} + +function createAwsFetchOnlyServices() { + const services = createBaseServices(); + const { config } = services; + + const awsOptions = createAwsOptions({ + user: config.get("awsUser"), + password: config.get("awsPassword"), + region: config.get("awsRegion"), + }); + const fetcher = new AwsMailFetcher(awsOptions, config.get("awsMailBucket")); + + return { ...services, awsOptions, fetcher }; +} + +function createAwsRecycleOnlyServices() { + const services = createAwsFetchOnlyServices(); + const { config } = services; + + const inbound = createInbound({ + fallback: config.getList("inboundFallback"), + ldaPath: config.get("ldaPath"), + doveadmPath: config.get("doveadmPath"), + aliasFile: join(config.get("dataPath"), "aliases.csv"), + mailDomain: config.get("mailDomain"), + }); + + return { ...services, inbound }; +} + +function createAwsServices() { + const services = createAwsRecycleOnlyServices(); + const { config, awsOptions, inbound } = services; + + const dbService = new DbService(join(config.get("dataPath"), "db.sqlite")); + const outbound = createOutbound(awsOptions, dbService, inbound); + + return { ...services, dbService, outbound }; +} + +function createServerServices() { + const services = createAwsServices(); + const { config, outbound, inbound, fetcher } = services; + + const smtp = createSmtp(outbound); + const hono = createHono(outbound, inbound); + + setupAwsHono(hono, { + path: config.get("awsInboundPath"), + auth: config.get("awsInboundKey"), + fetcher, + deliverer: inbound, + }); + + return { ...services, smtp, hono }; +} + +function serve(cron: boolean = false) { + const { config, fetcher, inbound, smtp, hono } = createServerServices(); + smtp.serve({ + hostname: config.get("smtpHost"), + port: config.getInt("smtpPort"), + }); + Deno.serve( + { + hostname: config.get("httpHost"), + port: config.getInt("httpPort"), + }, + hono.fetch, + ); + + if (cron) { + createCron(fetcher, inbound); + } +} + +async function listLives() { + const { fetcher } = createAwsFetchOnlyServices(); + const liveMails = await fetcher.listLiveMails(); + console.info(`Total ${liveMails.length}:`); + if (liveMails.length !== 0) { + console.info(liveMails.join("\n")); + } +} + +async function recycleLives() { + const { fetcher, inbound } = createAwsRecycleOnlyServices(); + await fetcher.recycleLiveMails(inbound); +} + +if (import.meta.main) { + await yargs(Deno.args) + .scriptName("mail") + .command({ + command: "sendmail", + describe: "send mail via this server's endpoint", + handler: async (_argv) => { + const { config } = createBaseServices(); + await sendMail(config.getInt("httpPort")); + }, + }) + .command({ + command: "live", + describe: "work with live mails", + builder: (builder) => { + return builder + .command({ + command: "list", + describe: "list live mails", + handler: listLives, + }) + .command({ + command: "recycle", + describe: "recycle all live mails", + handler: recycleLives, + }) + .demandCommand(1, "One command must be specified."); + }, + handler: () => {}, + }) + .command({ + command: "serve", + describe: "start the http and smtp servers", + builder: (builder) => builder.option("real", { type: "boolean" }), + handler: (argv) => serve(argv.real), + }) + .demandCommand(1, "One command must be specified.") + .help() + .strict() + .parse(); +} diff --git a/deno/mail/aws/deliver.ts b/deno/mail/aws/deliver.ts new file mode 100644 index 0000000..0195369 --- /dev/null +++ b/deno/mail/aws/deliver.ts @@ -0,0 +1,63 @@ +import { + SendEmailCommand, + SESv2Client, + SESv2ClientConfig, +} from "@aws-sdk/client-sesv2"; + +import { Mail, MailDeliverContext, MailDeliverer } from "../mail.ts"; + +declare module "../mail.ts" { + interface MailDeliverResult { + awsMessageId?: string; + } +} + +export class AwsMailDeliverer extends MailDeliverer { + readonly name = "aws"; + readonly #aws; + readonly #ses; + + constructor(aws: SESv2ClientConfig) { + super(true); + this.#aws = aws; + this.#ses = new SESv2Client(aws); + } + + protected override async doDeliver( + mail: Mail, + context: MailDeliverContext, + ): Promise { + try { + const sendCommand = new SendEmailCommand({ + Content: { + Raw: { Data: mail.toUtf8Bytes() }, + }, + }); + + console.info(context.logTag, "Calling aws send-email api..."); + const res = await this.#ses.send(sendCommand); + if (res.MessageId == null) { + console.warn( + context.logTag, + "AWS send-email returned null message id.", + ); + } else { + context.result.awsMessageId = + `${res.MessageId}@${this.#aws.region}.amazonses.com`; + } + + context.result.smtpMessage = + `AWS Message ID: ${context.result.awsMessageId}`; + context.result.recipients.set("*", { + kind: "success", + message: `Succeeded to call aws send-email api.`, + }); + } catch (cause) { + context.result.recipients.set("*", { + kind: "failure", + message: "A JS error was thrown when calling aws send-email." + cause, + cause, + }); + } + } +} diff --git a/deno/mail/aws/fetch.ts b/deno/mail/aws/fetch.ts new file mode 100644 index 0000000..2154972 --- /dev/null +++ b/deno/mail/aws/fetch.ts @@ -0,0 +1,136 @@ +import { + CopyObjectCommand, + DeleteObjectCommand, + GetObjectCommand, + ListObjectsV2Command, + NoSuchKey, + S3Client, + S3ClientConfig, +} from "@aws-sdk/client-s3"; + +import { DateUtils } from "@crupest/base"; + +import { Mail } from "../mail.ts"; +import { MailDeliverer } from "../mail.ts"; + +export class LiveMailNotFoundError extends Error {} + +async function s3MoveObject( + client: S3Client, + bucket: string, + path: string, + newPath: string, +): Promise { + const copyCommand = new CopyObjectCommand({ + Bucket: bucket, + Key: newPath, + CopySource: `${bucket}/${path}`, + }); + await client.send(copyCommand); + + const deleteCommand = new DeleteObjectCommand({ + Bucket: bucket, + Key: path, + }); + await client.send(deleteCommand); +} + +const AWS_SES_S3_SETUP_TAG = "AMAZON_SES_SETUP_NOTIFICATION"; + +export class AwsMailFetcher { + readonly #livePrefix = "mail/live/"; + readonly #archivePrefix = "mail/archive/"; + readonly #s3; + readonly #bucket; + + constructor(aws: S3ClientConfig, bucket: string) { + this.#s3 = new S3Client(aws); + this.#bucket = bucket; + } + + async listLiveMails(): Promise { + const listCommand = new ListObjectsV2Command({ + Bucket: this.#bucket, + Prefix: this.#livePrefix, + }); + const res = await this.#s3.send(listCommand); + + if (res.Contents == null) { + console.warn("S3 API returned null Content."); + return []; + } + + const result: string[] = []; + for (const object of res.Contents) { + if (object.Key == null) { + console.warn("S3 API returned null Key."); + continue; + } + + if (object.Key.endsWith(AWS_SES_S3_SETUP_TAG)) continue; + + result.push(object.Key.slice(this.#livePrefix.length)); + } + return result; + } + + async deliverLiveMail( + logTag: string, + s3Key: string, + deliverer: MailDeliverer, + recipients?: string[], + ) { + console.info(logTag, `Fetching live mail ${s3Key}...`); + const mailPath = `${this.#livePrefix}${s3Key}`; + const command = new GetObjectCommand({ + Bucket: this.#bucket, + Key: mailPath, + }); + + let rawMail; + + try { + const res = await this.#s3.send(command); + if (res.Body == null) { + throw new Error("S3 API returns a null body."); + } + rawMail = await res.Body.transformToString(); + } catch (cause) { + if (cause instanceof NoSuchKey) { + const message = + `Live mail ${s3Key} is not found. Perhaps already delivered?`; + console.error(message, cause); + throw new LiveMailNotFoundError(message); + } + throw cause; + } + + const mail = new Mail(rawMail); + await deliverer.deliver({ mail, recipients }); + + const { date } = new Mail(rawMail).parsed; + const dateString = date != null + ? DateUtils.toFileNameString(date, true) + : "invalid-date"; + const newPath = `${this.#archivePrefix}${dateString}/${s3Key}`; + + console.info(logTag, `Archiving live mail ${s3Key} to ${newPath}...`); + await s3MoveObject(this.#s3, this.#bucket, mailPath, newPath); + + console.info(logTag, `Done deliver live mail ${s3Key}.`); + } + + async recycleLiveMails(deliverer: MailDeliverer) { + console.info("Begin to recycle live mails..."); + const mails = await this.listLiveMails(); + console.info(`Found ${mails.length} live mails`); + let counter = 1; + for (const s3Key of mails) { + await this.deliverLiveMail( + `[${counter++}/${mails.length}]`, + s3Key, + deliverer, + ); + } + } +} diff --git a/deno/mail/aws/mail.ts b/deno/mail/aws/mail.ts new file mode 100644 index 0000000..26f3ea0 --- /dev/null +++ b/deno/mail/aws/mail.ts @@ -0,0 +1,59 @@ +import { MailDeliverContext, MailDeliverHook } from "../mail.ts"; + +export class AwsMailMessageIdRewriteHook implements MailDeliverHook { + readonly #lookup; + + constructor(lookup: (origin: string) => Promise) { + this.#lookup = lookup; + } + + async callback(context: MailDeliverContext): Promise { + const addresses = context.mail.simpleFindAllAddresses(); + for (const address of addresses) { + const awsMessageId = await this.#lookup(address); + if (awsMessageId != null && awsMessageId.length !== 0) { + console.info( + context.logTag, + `Rewrite address-line string in mail: ${address} => ${awsMessageId}.`, + ); + context.mail.raw = context.mail.raw.replaceAll(address, awsMessageId); + } + } + } +} + +export class AwsMailMessageIdSaveHook implements MailDeliverHook { + readonly #record; + + constructor( + record: ( + original: string, + aws: string, + context: MailDeliverContext, + ) => Promise, + ) { + this.#record = record; + } + + async callback(context: MailDeliverContext): Promise { + const { messageId } = context.mail.parsed; + if (messageId == null) { + console.warn( + context.logTag, + "Original mail doesn't have message id, skip saving message id map.", + ); + return; + } + if (context.result.awsMessageId != null) { + console.info( + context.logTag, + `Save message id map: ${messageId} => ${context.result.awsMessageId}.`, + ); + context.mail.raw = context.mail.raw.replaceAll( + messageId, + context.result.awsMessageId, + ); + await this.#record(messageId, context.result.awsMessageId, context); + } + } +} diff --git a/deno/mail/db.test.ts b/deno/mail/db.test.ts new file mode 100644 index 0000000..60035c4 --- /dev/null +++ b/deno/mail/db.test.ts @@ -0,0 +1,23 @@ +import { describe, it } from "@std/testing/bdd"; +import { expect } from "@std/expect/expect"; + +import { DbService } from "./db.ts"; + +describe("DbService", () => { + const mockRow = { + message_id: "mock-message-id@mock.mock", + aws_message_id: "mock-aws-message-id@mock.mock", + }; + + it("works", async () => { + const db = new DbService(":memory:"); + await db.migrate(); + await db.addMessageIdMap(mockRow); + expect(await db.messageIdToAws(mockRow.message_id)).toBe( + mockRow.aws_message_id, + ); + expect(await db.messageIdFromAws(mockRow.aws_message_id)).toBe( + mockRow.message_id, + ); + }); +}); diff --git a/deno/mail/db.ts b/deno/mail/db.ts new file mode 100644 index 0000000..062700b --- /dev/null +++ b/deno/mail/db.ts @@ -0,0 +1,146 @@ +import { + Generated, + Insertable, + Kysely, + Migration, + Migrator, + SqliteDatabase, + SqliteDialect, + SqliteStatement, +} from "kysely"; +import * as sqlite from "@db/sqlite"; + +class SqliteStatementAdapter implements SqliteStatement { + constructor(public readonly stmt: sqlite.Statement) {} + + get reader(): boolean { + return this.stmt.columnNames().length >= 1; + } + + all(parameters: readonly unknown[]): unknown[] { + return this.stmt.all(...(parameters as sqlite.BindValue[])); + } + + iterate(parameters: readonly unknown[]): IterableIterator { + return this.stmt.iter(...(parameters as sqlite.BindValue[])); + } + + run(parameters: readonly unknown[]): { + changes: number | bigint; + lastInsertRowid: number | bigint; + } { + const { db } = this.stmt; + const totalChangesBefore = db.totalChanges; + const changes = this.stmt.run(...(parameters as sqlite.BindValue[])); + return { + changes: totalChangesBefore === db.totalChanges ? 0 : changes, + lastInsertRowid: db.lastInsertRowId, + }; + } +} + +class SqliteDatabaseAdapter implements SqliteDatabase { + constructor(public readonly db: sqlite.Database) {} + + prepare(sql: string): SqliteStatementAdapter { + return new SqliteStatementAdapter(this.db.prepare(sql)); + } + + close(): void { + this.db.close(); + } +} + +export class DbError extends Error {} + +interface AwsMessageIdMapTable { + id: Generated; + message_id: string; + aws_message_id: string; +} + +interface Database { + aws_message_id_map: AwsMessageIdMapTable; +} + +const migrations: Record = { + "0001-init": { + // deno-lint-ignore no-explicit-any + async up(db: Kysely): Promise { + await db.schema + .createTable("aws_message_id_map") + .addColumn("id", "integer", (col) => col.primaryKey().autoIncrement()) + .addColumn("message_id", "text", (col) => col.notNull().unique()) + .addColumn("aws_message_id", "text", (col) => col.notNull().unique()) + .execute(); + + for (const column of ["message_id", "aws_message_id"]) { + await db.schema + .createIndex(`aws_message_id_map_${column}`) + .on("aws_message_id_map") + .column(column) + .execute(); + } + }, + + // deno-lint-ignore no-explicit-any + async down(db: Kysely): Promise { + await db.schema.dropTable("aws_message_id_map").execute(); + }, + }, +}; + +export class DbService { + #db; + #kysely; + #migrator; + + constructor(public readonly path: string) { + this.#db = new sqlite.Database(path); + this.#kysely = new Kysely({ + dialect: new SqliteDialect({ + database: new SqliteDatabaseAdapter(this.#db), + }), + }); + this.#migrator = new Migrator({ + db: this.#kysely, + provider: { + getMigrations(): Promise> { + return Promise.resolve(migrations); + }, + }, + }); + } + + async migrate(): Promise { + await this.#migrator.migrateToLatest(); + } + + async addMessageIdMap( + mail: Insertable, + ): Promise { + const inserted = await this.#kysely + .insertInto("aws_message_id_map") + .values(mail) + .executeTakeFirstOrThrow(); + return Number(inserted.insertId!); + } + + async messageIdToAws(messageId: string): Promise { + const row = await this.#kysely + .selectFrom("aws_message_id_map") + .where("message_id", "=", messageId) + .select("aws_message_id") + .executeTakeFirst(); + return row?.aws_message_id ?? null; + } + + async messageIdFromAws(awsMessageId: string): Promise { + const row = await this.#kysely + .selectFrom("aws_message_id_map") + .where("aws_message_id", "=", awsMessageId) + .select("message_id") + .executeTakeFirst(); + return row?.message_id ?? null; + } +} diff --git a/deno/mail/deno.json b/deno/mail/deno.json new file mode 100644 index 0000000..9105747 --- /dev/null +++ b/deno/mail/deno.json @@ -0,0 +1,18 @@ +{ + "version": "0.1.0", + "tasks": { + "run": "deno run -A aws/app.ts", + "compile": "deno compile -o out/crupest-relay -A aws/app.ts" + }, + "imports": { + "@aws-sdk/client-s3": "npm:@aws-sdk/client-s3@^3.821.0", + "@aws-sdk/client-sesv2": "npm:@aws-sdk/client-sesv2@^3.821.0", + "@db/sqlite": "jsr:@db/sqlite@^0.12.0", + "@hono/zod-validator": "npm:@hono/zod-validator@^0.7.0", + "@smithy/fetch-http-handler": "npm:@smithy/fetch-http-handler@^5.0.4", + "email-addresses": "npm:email-addresses@^5.0.0", + "hono": "npm:hono@^4.7.11", + "kysely": "npm:kysely@^0.28.2", + "zod": "npm:zod@^3.25.48" + } +} diff --git a/deno/mail/dovecot.ts b/deno/mail/dovecot.ts new file mode 100644 index 0000000..4fe2f41 --- /dev/null +++ b/deno/mail/dovecot.ts @@ -0,0 +1,213 @@ +import { Mail, MailDeliverContext, MailDeliverer } from "./mail.ts"; + +// https://doc.dovecot.org/main/core/man/dovecot-lda.1.html +const ldaExitCodeMessageMap = new Map(); +ldaExitCodeMessageMap.set(67, "recipient user not known"); +ldaExitCodeMessageMap.set(75, "temporary error"); + +type CommandResult = { + kind: "exit"; + status: Deno.CommandStatus; + logMessage: string; +} | { kind: "throw"; cause: unknown; logMessage: string }; + +async function runCommand( + bin: string, + options: { + logTag: string; + args: string[]; + stdin?: Uint8Array; + suppressResultLog?: boolean; + errorCodeMessageMap?: Map; + }, +): Promise { + const { logTag, args, stdin, suppressResultLog, errorCodeMessageMap } = + options; + + console.info(logTag, `Run external command ${bin} ${args.join(" ")}`); + + try { + // Create and spawn process. + const command = new Deno.Command(bin, { + args, + stdin: stdin == null ? "null" : "piped", + }); + const process = command.spawn(); + + // Write stdin if any. + if (stdin != null) { + const writer = process.stdin.getWriter(); + await writer.write(stdin); + writer.close(); + } + + // Wait for process to exit. + const status = await process.status; + + // Build log message string. + let message = `External command exited with code ${status.code}`; + if (status.signal != null) message += ` (signal: ${status.signal})`; + if (errorCodeMessageMap != null && errorCodeMessageMap.has(status.code)) { + message += `, ${errorCodeMessageMap.get(status.code)}`; + } + message += "."; + if (suppressResultLog !== true) console.log(logTag, message); + + // Return result. + return { + kind: "exit", + status, + logMessage: message, + }; + } catch (cause) { + const message = `A JS error was thrown when invoking external command:`; + if (suppressResultLog !== true) console.log(logTag, message); + return { kind: "throw", cause, logMessage: message + " " + cause }; + } +} + +export class DovecotMailDeliverer extends MailDeliverer { + readonly name = "dovecot"; + readonly #ldaPath; + readonly #doveadmPath; + + constructor( + ldaPath: string, + doveadmPath: string, + ) { + super(false); + this.#ldaPath = ldaPath; + this.#doveadmPath = doveadmPath; + } + + protected override async doDeliver( + mail: Mail, + context: MailDeliverContext, + ): Promise { + const utf8Bytes = mail.toUtf8Bytes(); + + const recipients = [...context.recipients]; + + if (recipients.length === 0) { + throw new Error( + "Failed to deliver to dovecot, no recipients are specified.", + ); + } + + for (const recipient of recipients) { + const result = await runCommand( + this.#ldaPath, + { + logTag: context.logTag, + args: ["-d", recipient], + stdin: utf8Bytes, + suppressResultLog: true, + errorCodeMessageMap: ldaExitCodeMessageMap, + }, + ); + + if (result.kind === "exit" && result.status.success) { + context.result.recipients.set(recipient, { + kind: "success", + message: result.logMessage, + }); + } else { + context.result.recipients.set(recipient, { + kind: "failure", + message: result.logMessage, + }); + } + } + } + + #queryArgs(mailbox: string, messageId: string) { + return ["mailbox", mailbox, "header", "Message-ID", `<${messageId}>`]; + } + + async #deleteMail( + logTag: string, + user: string, + mailbox: string, + messageId: string, + ): Promise { + await runCommand(this.#doveadmPath, { + logTag, + args: ["expunge", "-u", user, ...this.#queryArgs(mailbox, messageId)], + }); + } + + async #saveMail( + logTag: string, + user: string, + mailbox: string, + mail: Uint8Array, + ) { + await runCommand(this.#doveadmPath, { + logTag, + args: ["save", "-u", user, "-m", mailbox], + stdin: mail, + }); + } + + async #markAsRead( + logTag: string, + user: string, + mailbox: string, + messageId: string, + ) { + await runCommand(this.#doveadmPath, { + logTag, + args: [ + "flags", + "add", + "-u", + user, + "\\Seen", + ...this.#queryArgs(mailbox, messageId), + ], + }); + } + + async saveNewSent(logTag: string, mail: Mail, messageIdToDelete: string) { + console.info(logTag, "Save sent mail and delete ones with old message id."); + + // Try to get from and recipients from headers. + const { messageId, from, recipients } = mail.parsed; + + if (from == null) { + console.warn( + logTag, + "Failed to get sender (from) in headers, skip saving.", + ); + return; + } + + if (recipients.includes(from)) { + // So the mail should lie in the Inbox. + console.info( + logTag, + "One recipient of the mail is the sender itself, skip saving.", + ); + return; + } + + await this.#saveMail(logTag, from, "Sent", mail.toUtf8Bytes()); + if (messageId != null) { + await this.#markAsRead(logTag, from, "Sent", messageId); + } else { + console.warn( + "Message id of the mail is not found, skip marking as read.", + ); + } + + console.info( + logTag, + "Schedule deletion of old mails at 15,30,60 seconds later.", + ); + [15, 30, 60].forEach((seconds) => + setTimeout(() => { + void this.#deleteMail(logTag, from, "Sent", messageIdToDelete); + }, 1000 * seconds) + ); + } +} diff --git a/deno/mail/dumb-smtp-server.ts b/deno/mail/dumb-smtp-server.ts new file mode 100644 index 0000000..70d5ec0 --- /dev/null +++ b/deno/mail/dumb-smtp-server.ts @@ -0,0 +1,129 @@ +import { MailDeliverer } from "./mail.ts"; + +const CRLF = "\r\n"; + +function createResponses(host: string, port: number | string) { + const serverName = `[${host}]:${port}`; + return { + serverName, + READY: `220 ${serverName} SMTP Ready`, + EHLO: `250 ${serverName}`, + MAIL: "250 2.1.0 Sender OK", + RCPT: "250 2.1.5 Recipient OK", + DATA: "354 Start mail input; end with .", + QUIT: `211 2.0.0 ${serverName} closing connection`, + ACTIVE_CLOSE: "421 4.7.0 Please open a new connection to send more emails", + INVALID: "500 5.5.1 Error: command not recognized", + } as const; +} + +export class DumbSmtpServer { + #deliverer; + + constructor(deliverer: MailDeliverer) { + this.#deliverer = deliverer; + } + + async #handleConnection( + logTag: string, + conn: Deno.Conn, + responses: ReturnType, + ) { + using disposeStack = new DisposableStack(); + disposeStack.defer(() => { + console.info(logTag, "Close tcp connection."); + conn.close(); + }); + + console.info(logTag, "New tcp connection established."); + + const writer = conn.writable.getWriter(); + disposeStack.defer(() => writer.releaseLock()); + const reader = conn.readable.getReader(); + disposeStack.defer(() => reader.releaseLock()); + + const [decoder, encoder] = [new TextDecoder(), new TextEncoder()]; + const decode = (data: Uint8Array) => decoder.decode(data); + const send = async (s: string) => { + console.info(logTag, "Send line:", s); + await writer.write(encoder.encode(s + CRLF)); + }; + + let buffer: string = ""; + let rawMail: string | null = null; + + await send(responses["READY"]); + + while (true) { + const { value, done } = await reader.read(); + if (done) break; + + buffer += decode(value); + + while (true) { + const eolPos = buffer.indexOf(CRLF); + if (eolPos === -1) break; + + const line = buffer.slice(0, eolPos); + buffer = buffer.slice(eolPos + CRLF.length); + + if (rawMail == null) { + console.info(logTag, "Received line:", line); + const upperLine = line.toUpperCase(); + if (upperLine.startsWith("EHLO") || upperLine.startsWith("HELO")) { + await send(responses["EHLO"]); + } else if (upperLine.startsWith("MAIL FROM:")) { + await send(responses["MAIL"]); + } else if (upperLine.startsWith("RCPT TO:")) { + await send(responses["RCPT"]); + } else if (upperLine === "DATA") { + await send(responses["DATA"]); + console.info(logTag, "Begin to receive mail data..."); + rawMail = ""; + } else if (upperLine === "QUIT") { + await send(responses["QUIT"]); + return; + } else { + await send(responses["INVALID"]); + return; + } + } else { + if (line === ".") { + try { + console.info(logTag, "Mail data received, begin to relay..."); + const { smtpMessage } = await this.#deliverer.deliverRaw(rawMail); + await send(`250 2.6.0 ${smtpMessage}`); + rawMail = null; + } catch (err) { + console.error(logTag, "Relay failed.", err); + await send("554 5.3.0 Error: check server log"); + } + await send(responses["ACTIVE_CLOSE"]); + } else { + const dataLine = line.startsWith("..") ? line.slice(1) : line; + rawMail += dataLine + CRLF; + } + } + } + } + } + + async serve(options: { hostname: string; port: number }) { + const listener = Deno.listen(options); + const responses = createResponses(options.hostname, options.port); + console.info( + `Dumb SMTP server starts to listen on ${responses.serverName}.`, + ); + + let counter = 1; + + for await (const conn of listener) { + const logTag = `[outbound ${counter++}]`; + try { + await this.#handleConnection(logTag, conn, responses); + } catch (cause) { + console.error(logTag, "A JS error was thrown by handler:", cause); + } + } + } +} diff --git a/deno/mail/mail-parsing.ts b/deno/mail/mail-parsing.ts new file mode 100644 index 0000000..8e9697d --- /dev/null +++ b/deno/mail/mail-parsing.ts @@ -0,0 +1,144 @@ +import emailAddresses from "email-addresses"; + +class MailParsingError extends Error {} + +function parseHeaderSection(section: string) { + const headers = [] as [key: string, value: string][]; + + let field: string | null = null; + let lineNumber = 1; + + const handleField = () => { + if (field == null) return; + const sepPos = field.indexOf(":"); + if (sepPos === -1) { + throw new MailParsingError( + `Expect ':' in the header field line: ${field}`, + ); + } + headers.push([field.slice(0, sepPos).trim(), field.slice(sepPos + 1)]); + field = null; + }; + + for (const line of section.trimEnd().split(/\r?\n|\r/)) { + if (line.match(/^\s/)) { + if (field == null) { + throw new MailParsingError("Header section starts with a space."); + } + field += line; + } else { + handleField(); + field = line; + } + lineNumber += 1; + } + + handleField(); + + return headers; +} + +function findFirst(fields: readonly [string, string][], key: string) { + for (const [k, v] of fields) { + if (key.toLowerCase() === k.toLowerCase()) return v; + } + return undefined; +} + +function findMessageId(fields: readonly [string, string][]) { + const messageIdField = findFirst(fields, "message-id"); + if (messageIdField == null) return undefined; + + const match = messageIdField.match(/\<(.*?)\>/); + if (match != null) { + return match[1]; + } else { + console.warn(`Invalid syntax in header 'message-id': ${messageIdField}`); + return undefined; + } +} + +function findDate(fields: readonly [string, string][]) { + const dateField = findFirst(fields, "date"); + if (dateField == null) return undefined; + + const date = new Date(dateField); + if (isNaN(date.getTime())) { + console.warn(`Invalid date string in header 'date': ${dateField}`); + return undefined; + } + return date; +} + +function findFrom(fields: readonly [string, string][]) { + const fromField = findFirst(fields, "from"); + if (fromField == null) return undefined; + + const addr = emailAddresses.parseOneAddress(fromField); + return addr?.type === "mailbox" ? addr.address : undefined; +} + +function findRecipients(fields: readonly [string, string][]) { + const headers = ["to", "cc", "bcc", "x-original-to"]; + const recipients = new Set(); + for (const [key, value] of fields) { + if (headers.includes(key.toLowerCase())) { + emailAddresses + .parseAddressList(value) + ?.flatMap((a) => (a.type === "mailbox" ? a : a.addresses)) + ?.forEach(({ address }) => recipients.add(address)); + } + } + return recipients; +} + +function parseSections(raw: string) { + const twoEolMatch = raw.match(/(\r?\n)(\r?\n)/); + if (twoEolMatch == null) { + throw new MailParsingError( + "No header/body section separator (2 successive EOLs) found.", + ); + } + + const [eol, sep] = [twoEolMatch[1], twoEolMatch[2]]; + + if (eol !== sep) { + console.warn("Different EOLs (\\r\\n, \\n) found."); + } + + return { + header: raw.slice(0, twoEolMatch.index!), + body: raw.slice(twoEolMatch.index! + eol.length + sep.length), + eol, + sep, + }; +} + +export type ParsedMail = Readonly<{ + header: string; + body: string; + sep: string; + eol: string; + headers: readonly [string, string][]; + messageId: string | undefined; + date: Date | undefined; + from: string | undefined; + recipients: readonly string[]; +}>; + +export function simpleParseMail(raw: string): ParsedMail { + const sections = Object.freeze(parseSections(raw)); + const headers = Object.freeze(parseHeaderSection(sections.header)); + const messageId = findMessageId(headers); + const date = findDate(headers); + const from = findFrom(headers); + const recipients = Object.freeze([...findRecipients(headers)]); + return Object.freeze({ + ...sections, + headers, + messageId, + date, + from, + recipients, + }); +} diff --git a/deno/mail/mail.test.ts b/deno/mail/mail.test.ts new file mode 100644 index 0000000..a8204be --- /dev/null +++ b/deno/mail/mail.test.ts @@ -0,0 +1,121 @@ +import { describe, it } from "@std/testing/bdd"; +import { expect, fn } from "@std/expect"; + +import { Mail, MailDeliverContext, MailDeliverer } from "./mail.ts"; + +const mockDate = "Fri, 02 May 2025 08:33:02 +0000"; +const mockMessageId = "mock-message-id@from.mock"; +const mockMessageId2 = "mock-message-id-2@from.mock"; +const mockFromAddress = "mock@from.mock"; +const mockCcAddress = "mock@cc.mock"; +const mockBodyStr = `This is body content. +Line 2 ${mockMessageId2} + +Line 4`; +const mockHeaders = [ + ["Content-Disposition", "inline"], + ["Content-Transfer-Encoding", "quoted-printable"], + ["MIME-Version", "1.0"], + ["X-Mailer", "MIME-tools 5.509 (Entity 5.509)"], + ["Content-Type", "text/plain; charset=utf-8"], + ["From", `"Mock From" <${mockFromAddress}>`], + [ + "To", + `"John \\"Big\\" Doe" , "Alice (Work)" , + undisclosed-recipients:;, "Group: Team" , + "Escaped, Name" , just@email.com, + "Comment (This is valid)" , + "Odd @Chars" , + "Non-ASCII 用户" , + admin@[192.168.1.1]`, + ], + ["CC", `Mock CC <${mockCcAddress}>`], + ["Subject", "A very long mock\n subject"], + ["Message-ID", `<${mockMessageId}>`], + ["Date", mockDate], +]; +const mockHeaderStr = mockHeaders.map((h) => h[0] + ": " + h[1]).join("\n"); +const mockMailStr = mockHeaderStr + "\n\n" + mockBodyStr; +const mockCrlfMailStr = mockMailStr.replaceAll("\n", "\r\n"); +const mockToAddresses = [ + "john@example.com", + "alice+work@example.com", + "team@company.com", + "escape@test.com", + "just@email.com", + "comment@domain.net", + "weird!#$%'*+-/=?^_`{|}~@char-test.com", + "user@例子.中国", + "admin@[192.168.1.1]", +]; + +describe("Mail", () => { + it("simple parse", () => { + const { parsed } = new Mail(mockMailStr); + expect(parsed.header).toEqual(mockHeaderStr); + expect(parsed.body).toEqual(mockBodyStr); + expect(parsed.sep).toBe("\n"); + expect(parsed.eol).toBe("\n"); + }); + + it("simple parse crlf", () => { + const { parsed } = new Mail(mockCrlfMailStr); + expect(parsed.sep).toBe("\r\n"); + expect(parsed.eol).toBe("\r\n"); + }); + + it("simple parse date", () => { + expect( + new Mail(mockMailStr).parsed.date, + ).toEqual(new Date(mockDate)); + }); + + it("simple parse headers", () => { + expect( + new Mail(mockMailStr).parsed.headers, + ).toEqual(mockHeaders.map((h) => [h[0], " " + h[1].replaceAll("\n", "")])); + }); + + it("parse recipients", () => { + const mail = new Mail(mockMailStr); + expect([...mail.parsed.recipients]).toEqual([ + ...mockToAddresses, + mockCcAddress, + ]); + }); + + it("find all addresses", () => { + const mail = new Mail(mockMailStr); + expect(mail.simpleFindAllAddresses()).toEqual([ + "mock@from.mock", + "john@example.com", + "alice+work@example.com", + "team@company.com", + "escape@test.com", + "just@email.com", + "comment@domain.net", + "mock@cc.mock", + "mock-message-id@from.mock", + "mock-message-id-2@from.mock", + ]); + }); +}); + +describe("MailDeliverer", () => { + class MockMailDeliverer extends MailDeliverer { + name = "mock"; + override doDeliver = fn((_: Mail, ctx: MailDeliverContext) => { + ctx.result.recipients.set("*", { + kind: "success", + message: "success message", + }); + return Promise.resolve(); + }) as MailDeliverer["doDeliver"]; + } + const mockDeliverer = new MockMailDeliverer(false); + + it("deliver success", async () => { + await mockDeliverer.deliverRaw(mockMailStr); + expect(mockDeliverer.doDeliver).toHaveBeenCalledTimes(1); + }); +}); diff --git a/deno/mail/mail.ts b/deno/mail/mail.ts new file mode 100644 index 0000000..9cc591c --- /dev/null +++ b/deno/mail/mail.ts @@ -0,0 +1,234 @@ +import { encodeBase64 } from "@std/encoding/base64"; +import { parse } from "@std/csv/parse"; +import { simpleParseMail } from "./mail-parsing.ts"; + +export class Mail { + #raw; + #parsed; + + constructor(raw: string) { + this.#raw = raw; + this.#parsed = simpleParseMail(raw); + } + + get raw() { + return this.#raw; + } + + set raw(value) { + this.#raw = value; + this.#parsed = simpleParseMail(value); + } + + get parsed() { + return this.#parsed; + } + + toUtf8Bytes(): Uint8Array { + const utf8Encoder = new TextEncoder(); + return utf8Encoder.encode(this.raw); + } + + toBase64(): string { + return encodeBase64(this.raw); + } + + simpleFindAllAddresses(): string[] { + const re = /,?\?,?/gi; + return [...this.raw.matchAll(re)].map((m) => m[1]); + } +} + +export interface MailDeliverRecipientResult { + kind: "success" | "failure"; + message?: string; + cause?: unknown; +} + +export class MailDeliverResult { + message?: string; + smtpMessage?: string; + recipients = new Map(); + constructor(public mail: Mail) {} + + get hasFailure() { + return this.recipients.values().some((v) => v.kind !== "success"); + } + + generateLogMessage(prefix: string) { + const lines = []; + if (this.message != null) lines.push(`${prefix} message: ${this.message}`); + if (this.smtpMessage != null) { + lines.push(`${prefix} smtpMessage: ${this.smtpMessage}`); + } + for (const [name, result] of this.recipients.entries()) { + const { kind, message } = result; + lines.push(`${prefix} (${name}): ${kind} ${message}`); + } + return lines.join("\n"); + } +} + +export class MailDeliverContext { + readonly recipients: Set = new Set(); + readonly result; + + constructor(public logTag: string, public mail: Mail) { + this.result = new MailDeliverResult(this.mail); + } +} + +export interface MailDeliverHook { + callback(context: MailDeliverContext): Promise; +} + +export abstract class MailDeliverer { + #counter = 1; + #last?: Promise; + + abstract name: string; + preHooks: MailDeliverHook[] = []; + postHooks: MailDeliverHook[] = []; + + constructor(public sync: boolean) {} + + protected abstract doDeliver( + mail: Mail, + context: MailDeliverContext, + ): Promise; + + async deliverRaw(rawMail: string) { + return await this.deliver({ mail: new Mail(rawMail) }); + } + + async #deliverCore(context: MailDeliverContext) { + for (const hook of this.preHooks) { + await hook.callback(context); + } + + await this.doDeliver(context.mail, context); + + for (const hook of this.postHooks) { + await hook.callback(context); + } + } + + async deliver(options: { + mail: Mail; + recipients?: string[]; + logTag?: string; + }): Promise { + const logTag = options.logTag ?? `[${this.name} ${this.#counter}]`; + this.#counter++; + + if (this.#last != null) { + console.info(logTag, "Wait for last delivering done..."); + await this.#last; + } + + const context = new MailDeliverContext( + logTag, + options.mail, + ); + options.recipients?.forEach((r) => context.recipients.add(r)); + + console.info(context.logTag, "Begin to deliver mail..."); + + const deliverPromise = this.#deliverCore(context); + + if (this.sync) { + this.#last = deliverPromise.then(() => {}, () => {}); + } + + await deliverPromise; + this.#last = undefined; + + console.info(context.logTag, "Deliver result:"); + console.info(context.result.generateLogMessage(context.logTag)); + + if (context.result.hasFailure) { + throw new Error("Failed to deliver to some recipients."); + } + + return context.result; + } +} + +export class RecipientFromHeadersHook implements MailDeliverHook { + constructor(public mailDomain: string) {} + + callback(context: MailDeliverContext) { + if (context.recipients.size !== 0) { + console.warn( + context.logTag, + "Recipients are already filled, skip inferring from headers.", + ); + } else { + [...context.mail.parsed.recipients].filter((r) => + r.endsWith("@" + this.mailDomain) + ).forEach((r) => context.recipients.add(r)); + + console.info( + context.logTag, + "Use recipients inferred from mail headers:", + [...context.recipients].join(", "), + ); + } + return Promise.resolve(); + } +} + +export class FallbackRecipientHook implements MailDeliverHook { + constructor(public fallback: Set = new Set()) {} + + callback(context: MailDeliverContext) { + if (context.recipients.size === 0) { + console.info( + context.logTag, + "Use fallback recipients:" + [...this.fallback].join(", "), + ); + this.fallback.forEach((a) => context.recipients.add(a)); + } + return Promise.resolve(); + } +} + +export class AliasRecipientMailHook implements MailDeliverHook { + #aliasFile; + + constructor(aliasFile: string) { + this.#aliasFile = aliasFile; + } + + async #parseAliasFile(logTag: string): Promise> { + const result = new Map(); + if ((await Deno.stat(this.#aliasFile)).isFile) { + const text = await Deno.readTextFile(this.#aliasFile); + const csv = parse(text); + for (const [real, ...aliases] of csv) { + aliases.forEach((a) => result.set(a, real)); + } + } else { + console.warn( + logTag, + `Recipient alias file ${this.#aliasFile} is not found.`, + ); + } + return result; + } + + async callback(context: MailDeliverContext) { + const aliases = await this.#parseAliasFile(context.logTag); + for (const recipient of [...context.recipients]) { + const realRecipients = aliases.get(recipient); + if (realRecipients != null) { + console.info( + context.logTag, + `Recipient alias resolved: ${recipient} => ${realRecipients}.`, + ); + context.recipients.delete(recipient); + context.recipients.add(realRecipients); + } + } + } +} -- cgit v1.2.3