aboutsummaryrefslogtreecommitdiff
path: root/deno
diff options
context:
space:
mode:
Diffstat (limited to 'deno')
-rw-r--r--deno/base/config.ts6
-rw-r--r--deno/base/deno.json3
-rw-r--r--deno/base/lib.ts24
-rw-r--r--deno/base/log.ts60
-rw-r--r--deno/deno.json7
-rw-r--r--deno/deno.lock7
-rw-r--r--deno/mail-relay/aws/mail.ts49
-rw-r--r--deno/mail-relay/dovecot.ts99
-rw-r--r--deno/mail-relay/mail.ts330
-rw-r--r--deno/mail/app.ts (renamed from deno/mail-relay/app.ts)17
-rw-r--r--deno/mail/aws/app.ts (renamed from deno/mail-relay/aws/app.ts)98
-rw-r--r--deno/mail/aws/deliver.ts (renamed from deno/mail-relay/aws/deliver.ts)35
-rw-r--r--deno/mail/aws/fetch.ts (renamed from deno/mail-relay/aws/fetch.ts)80
-rw-r--r--deno/mail/db.test.ts (renamed from deno/mail-relay/db.test.ts)8
-rw-r--r--deno/mail/db.ts (renamed from deno/mail-relay/db.ts)36
-rw-r--r--deno/mail/deno.json (renamed from deno/mail-relay/deno.json)2
-rw-r--r--deno/mail/dovecot.ts219
-rw-r--r--deno/mail/dumb-smtp-server.ts (renamed from deno/mail-relay/dumb-smtp-server.ts)61
-rw-r--r--deno/mail/mail-parsing.ts144
-rw-r--r--deno/mail/mail.test.ts (renamed from deno/mail-relay/mail.test.ts)31
-rw-r--r--deno/mail/mail.ts304
-rw-r--r--deno/tools/deno.json2
-rw-r--r--deno/tools/geosite.ts (renamed from deno/tools/generate-geosite-rules.ts)7
-rw-r--r--deno/tools/main.ts14
-rw-r--r--deno/tools/manage-service.ts42
-rw-r--r--deno/tools/manage-vm.ts144
-rw-r--r--deno/tools/service.ts180
-rw-r--r--deno/tools/template.ts122
-rw-r--r--deno/tools/vm.ts225
-rw-r--r--deno/tools/yargs.ts12
30 files changed, 1328 insertions, 1040 deletions
diff --git a/deno/base/config.ts b/deno/base/config.ts
index a5f5d86..96cc869 100644
--- a/deno/base/config.ts
+++ b/deno/base/config.ts
@@ -1,4 +1,4 @@
-import { camelCaseToKebabCase } from "./lib.ts";
+import { StringUtils } from "./lib.ts";
export interface ConfigDefinitionItem {
readonly description: string;
@@ -29,7 +29,9 @@ export class ConfigProvider<K extends string> {
for (const [key, def] of Object.entries(definition as ConfigDefinition)) {
map[key] = {
...def,
- env: `${this.#prefix}-${camelCaseToKebabCase(key as string)}`
+ env: `${this.#prefix}-${
+ StringUtils.camelCaseToKebabCase(key as string)
+ }`
.replaceAll("-", "_")
.toUpperCase(),
};
diff --git a/deno/base/deno.json b/deno/base/deno.json
index dabc02a..582f0f6 100644
--- a/deno/base/deno.json
+++ b/deno/base/deno.json
@@ -4,7 +4,6 @@
"exports": {
".": "./lib.ts",
"./config": "./config.ts",
- "./cron": "./cron.ts",
- "./log": "./log.ts"
+ "./cron": "./cron.ts"
}
}
diff --git a/deno/base/lib.ts b/deno/base/lib.ts
index a5e4a6a..af75115 100644
--- a/deno/base/lib.ts
+++ b/deno/base/lib.ts
@@ -1,10 +1,30 @@
-export function camelCaseToKebabCase(str: string): string {
+function camelCaseToKebabCase(str: string): string {
return str.replace(/[A-Z]/g, (m) => "-" + m.toLowerCase());
}
-export function toFileNameString(date: Date, dateOnly?: boolean): string {
+function prependNonEmpty<T>(
+ object: T | null | undefined,
+ prefix: string = " ",
+): string {
+ if (object == null) return "";
+ const string = typeof object === "string" ? object : String(object);
+ return string.length === 0 ? "" : prefix + string;
+}
+
+export const StringUtils = Object.freeze({
+ camelCaseToKebabCase,
+ prependNonEmpty,
+});
+
+function toFileNameString(date: Date, dateOnly?: boolean): string {
const str = date.toISOString();
return dateOnly === true
? str.slice(0, str.indexOf("T"))
: str.replaceAll(/:|\./g, "-");
}
+
+export const DateUtils = Object.freeze(
+ {
+ toFileNameString,
+ } as const,
+);
diff --git a/deno/base/log.ts b/deno/base/log.ts
deleted file mode 100644
index 940f569..0000000
--- a/deno/base/log.ts
+++ /dev/null
@@ -1,60 +0,0 @@
-import { join } from "@std/path";
-
-import { toFileNameString } from "./lib.ts";
-
-export interface ExternalLogStream extends Disposable {
- stream: WritableStream;
-}
-
-export class LogFileProvider {
- #directory: string;
-
- constructor(directory: string) {
- this.#directory = directory;
- Deno.mkdirSync(directory, { recursive: true });
- }
-
- async createExternalLogStream(
- name: string,
- options?: {
- noTime?: boolean;
- },
- ): Promise<ExternalLogStream> {
- if (name.includes("/")) {
- throw new Error(`External log stream's name (${name}) contains '/'.`);
- }
-
- const logPath = join(
- this.#directory,
- options?.noTime === true
- ? name
- : `${name}-${toFileNameString(new Date())}`,
- );
-
- const file = await Deno.open(logPath, {
- read: false,
- write: true,
- append: true,
- create: true,
- });
- return {
- stream: file.writable,
- [Symbol.dispose]: file[Symbol.dispose].bind(file),
- };
- }
-
- async createExternalLogStreamsForProgram(
- program: string,
- ): Promise<{ stdout: WritableStream; stderr: WritableStream } & Disposable> {
- const stdout = await this.createExternalLogStream(`${program}-stdout`);
- const stderr = await this.createExternalLogStream(`${program}-stderr`);
- return {
- stdout: stdout.stream,
- stderr: stderr.stream,
- [Symbol.dispose]: () => {
- stdout[Symbol.dispose]();
- stderr[Symbol.dispose]();
- },
- };
- }
-}
diff --git a/deno/deno.json b/deno/deno.json
index f4859d1..286451e 100644
--- a/deno/deno.json
+++ b/deno/deno.json
@@ -1,7 +1,7 @@
{
- "workspace": ["./base", "./mail-relay", "./tools" ],
+ "workspace": ["./base", "./mail", "./tools"],
"tasks": {
- "compile:mail-relay": "deno task --cwd=mail-relay compile",
+ "compile:mail": "deno task --cwd=mail compile"
},
"imports": {
"@std/collections": "jsr:@std/collections@^1.1.1",
@@ -11,9 +11,8 @@
"@std/io": "jsr:@std/io@^0.225.2",
"@std/path": "jsr:@std/path@^1.1.0",
"@std/testing": "jsr:@std/testing@^1.0.13",
- "@std/dotenv": "jsr:@std/dotenv@^0.225.5",
"@std/fs": "jsr:@std/fs@^1.0.18",
- "yargs": "npm:yargs@^18.0.0"
+ "yargs": "npm:yargs@^18.0.0",
"@types/yargs": "npm:@types/yargs@^17.0.33"
}
}
diff --git a/deno/deno.lock b/deno/deno.lock
index 871a9ae..bdc8c3f 100644
--- a/deno/deno.lock
+++ b/deno/deno.lock
@@ -10,7 +10,6 @@
"jsr:@std/collections@^1.1.1": "1.1.1",
"jsr:@std/csv@^1.0.6": "1.0.6",
"jsr:@std/data-structures@^1.0.8": "1.0.8",
- "jsr:@std/dotenv@~0.225.5": "0.225.5",
"jsr:@std/encoding@1": "1.0.10",
"jsr:@std/encoding@^1.0.10": "1.0.10",
"jsr:@std/expect@^1.0.16": "1.0.16",
@@ -88,9 +87,6 @@
"@std/data-structures@1.0.8": {
"integrity": "2fb7219247e044c8fcd51341788547575653c82ae2c759ff209e0263ba7d9b66"
},
- "@std/dotenv@0.225.5": {
- "integrity": "9ce6f9d0ec3311f74a32535aa1b8c62ed88b1ab91b7f0815797d77a6f60c922f"
- },
"@std/encoding@1.0.10": {
"integrity": "8783c6384a2d13abd5e9e87a7ae0520a30e9f56aeeaa3bdf910a3eaaf5c811a1"
},
@@ -1300,7 +1296,6 @@
"dependencies": [
"jsr:@std/collections@^1.1.1",
"jsr:@std/csv@^1.0.6",
- "jsr:@std/dotenv@~0.225.5",
"jsr:@std/encoding@^1.0.10",
"jsr:@std/expect@^1.0.16",
"jsr:@std/fs@^1.0.18",
@@ -1311,7 +1306,7 @@
"npm:yargs@18"
],
"members": {
- "mail-relay": {
+ "mail": {
"dependencies": [
"jsr:@db/sqlite@0.12",
"npm:@aws-sdk/client-s3@^3.821.0",
diff --git a/deno/mail-relay/aws/mail.ts b/deno/mail-relay/aws/mail.ts
deleted file mode 100644
index cc05d23..0000000
--- a/deno/mail-relay/aws/mail.ts
+++ /dev/null
@@ -1,49 +0,0 @@
-import { MailDeliverContext, MailDeliverHook } from "../mail.ts";
-
-export class AwsMailMessageIdRewriteHook implements MailDeliverHook {
- readonly #lookup;
-
- constructor(lookup: (origin: string) => Promise<string | null>) {
- this.#lookup = lookup;
- }
-
- async callback(context: MailDeliverContext): Promise<void> {
- console.info("Rewrite message ids...");
- const addresses = context.mail.simpleFindAllAddresses();
- console.info(`Addresses found in mail: ${addresses.join(", ")}.`);
- for (const address of addresses) {
- const awsMessageId = await this.#lookup(address);
- if (awsMessageId != null && awsMessageId.length !== 0) {
- console.info(`Rewrite ${address} to ${awsMessageId}.`);
- context.mail.raw = context.mail.raw.replaceAll(address, awsMessageId);
- }
- }
- console.info("Done rewrite message ids.");
- }
-}
-
-export class AwsMailMessageIdSaveHook implements MailDeliverHook {
- readonly #record;
-
- constructor(record: (original: string, aws: string) => Promise<void>) {
- this.#record = record;
- }
-
- async callback(context: MailDeliverContext): Promise<void> {
- console.info("Save aws message ids...");
- const messageId = context.mail
- .startSimpleParse()
- .sections()
- .headers()
- .messageId();
- if (messageId == null) {
- console.info("Original mail does not have message id. Skip saving.");
- return;
- }
- if (context.result.awsMessageId != null) {
- console.info(`Saving ${messageId} => ${context.result.awsMessageId}.`);
- await this.#record(messageId, context.result.awsMessageId);
- }
- console.info("Done save message ids.");
- }
-}
diff --git a/deno/mail-relay/dovecot.ts b/deno/mail-relay/dovecot.ts
deleted file mode 100644
index 333b803..0000000
--- a/deno/mail-relay/dovecot.ts
+++ /dev/null
@@ -1,99 +0,0 @@
-import { basename } from "@std/path";
-
-import { LogFileProvider } from "@crupest/base/log";
-
-import { Mail, MailDeliverContext, MailDeliverer } from "./mail.ts";
-
-export class DovecotMailDeliverer extends MailDeliverer {
- readonly name = "dovecot";
- readonly #logFileProvider;
- readonly #ldaPath;
-
- constructor(logFileProvider: LogFileProvider, ldaPath: string) {
- super();
- this.#logFileProvider = logFileProvider;
- this.#ldaPath = ldaPath;
- }
-
- protected override async doDeliver(
- mail: Mail,
- context: MailDeliverContext,
- ): Promise<void> {
- const ldaPath = this.#ldaPath;
- const ldaBinName = basename(ldaPath);
- const utf8Stream = mail.toUtf8Bytes();
-
- const recipients = [...context.recipients];
-
- if (recipients.length === 0) {
- context.result.message =
- "Failed to deliver to dovecot, no recipients are specified.";
- return;
- }
-
- console.info(`Deliver to dovecot users: ${recipients.join(", ")}.`);
-
- for (const recipient of recipients) {
- try {
- const commandArgs = ["-d", recipient];
- console.info(`Run ${ldaBinName} ${commandArgs.join(" ")}...`);
-
- const ldaCommand = new Deno.Command(ldaPath, {
- args: commandArgs,
- stdin: "piped",
- stdout: "piped",
- stderr: "piped",
- });
-
- const ldaProcess = ldaCommand.spawn();
- using logFiles =
- await this.#logFileProvider.createExternalLogStreamsForProgram(
- ldaBinName,
- );
- ldaProcess.stdout.pipeTo(logFiles.stdout);
- ldaProcess.stderr.pipeTo(logFiles.stderr);
-
- const stdinWriter = ldaProcess.stdin.getWriter();
- await stdinWriter.write(utf8Stream);
- await stdinWriter.close();
-
- const status = await ldaProcess.status;
-
- if (status.success) {
- context.result.recipients.set(recipient, {
- kind: "done",
- message: `${ldaBinName} exited with success.`,
- });
- } else {
- let message = `${ldaBinName} exited with error code ${status.code}`;
-
- if (status.signal != null) {
- message += ` (signal ${status.signal})`;
- }
-
- // https://doc.dovecot.org/main/core/man/dovecot-lda.1.html
- switch (status.code) {
- case 67:
- message += ", recipient user not known";
- break;
- case 75:
- message += ", temporary error";
- break;
- }
-
- message += ".";
-
- context.result.recipients.set(recipient, { kind: "fail", message });
- }
- } catch (cause) {
- context.result.recipients.set(recipient, {
- kind: "fail",
- message: "An error is thrown when running lda: " + cause,
- cause,
- });
- }
- }
-
- console.info("Done handling all recipients.");
- }
-}
diff --git a/deno/mail-relay/mail.ts b/deno/mail-relay/mail.ts
deleted file mode 100644
index d6dfe65..0000000
--- a/deno/mail-relay/mail.ts
+++ /dev/null
@@ -1,330 +0,0 @@
-import { encodeBase64 } from "@std/encoding/base64";
-import { parse } from "@std/csv/parse";
-import emailAddresses from "email-addresses";
-
-class MailSimpleParseError extends Error {}
-
-class MailSimpleParsedHeaders {
- constructor(public fields: [key: string, value: string][]) {}
-
- getFirst(fieldKey: string): string | undefined {
- for (const [key, value] of this.fields) {
- if (key.toLowerCase() === fieldKey.toLowerCase()) return value;
- }
- return undefined;
- }
-
- messageId(): string | undefined {
- const messageIdField = this.getFirst("message-id");
- if (messageIdField == null) return undefined;
-
- const match = messageIdField.match(/\<(.*?)\>/);
- if (match != null) {
- return match[1];
- } else {
- console.warn("Invalid message-id header of mail: " + messageIdField);
- return undefined;
- }
- }
-
- date(invalidToUndefined: boolean = true): Date | undefined {
- const dateField = this.getFirst("date");
- if (dateField == null) return undefined;
-
- const date = new Date(dateField);
- if (invalidToUndefined && isNaN(date.getTime())) {
- console.warn(`Invalid date string (${dateField}) found in header.`);
- return undefined;
- }
- return date;
- }
-
- recipients(options?: { domain?: string; headers?: string[] }): Set<string> {
- const domain = options?.domain;
- const headers = options?.headers ?? ["to", "cc", "bcc", "x-original-to"];
- const recipients = new Set<string>();
- for (const [key, value] of this.fields) {
- if (headers.includes(key.toLowerCase())) {
- emailAddresses
- .parseAddressList(value)
- ?.flatMap((a) => (a.type === "mailbox" ? a : a.addresses))
- ?.forEach(({ address }) => {
- if (domain == null || address.endsWith(domain)) {
- recipients.add(address);
- }
- });
- }
- }
- return recipients;
- }
-}
-
-class MailSimpleParsedSections {
- header: string;
- body: string;
- eol: string;
- sep: string;
-
- constructor(raw: string) {
- const twoEolMatch = raw.match(/(\r?\n)(\r?\n)/);
- if (twoEolMatch == null) {
- throw new MailSimpleParseError(
- "No header/body section separator (2 successive EOLs) found.",
- );
- }
-
- const [eol, sep] = [twoEolMatch[1], twoEolMatch[2]];
-
- if (eol !== sep) {
- console.warn("Different EOLs (\\r\\n, \\n) found.");
- }
-
- this.header = raw.slice(0, twoEolMatch.index!);
- this.body = raw.slice(twoEolMatch.index! + eol.length + sep.length);
- this.eol = eol;
- this.sep = sep;
- }
-
- headers(): MailSimpleParsedHeaders {
- const headers = [] as [key: string, value: string][];
-
- let field: string | null = null;
- let lineNumber = 1;
-
- const handleField = () => {
- if (field == null) return;
- const sepPos = field.indexOf(":");
- if (sepPos === -1) {
- throw new MailSimpleParseError(`No ':' in the header line: ${field}`);
- }
- headers.push([field.slice(0, sepPos).trim(), field.slice(sepPos + 1)]);
- field = null;
- };
-
- for (const line of this.header.trimEnd().split(/\r?\n|\r/)) {
- if (line.match(/^\s/)) {
- if (field == null) {
- throw new MailSimpleParseError("Header section starts with a space.");
- }
- field += line;
- } else {
- handleField();
- field = line;
- }
- lineNumber += 1;
- }
-
- handleField();
-
- return new MailSimpleParsedHeaders(headers);
- }
-}
-
-export class Mail {
- constructor(public raw: string) {}
-
- toUtf8Bytes(): Uint8Array {
- const utf8Encoder = new TextEncoder();
- return utf8Encoder.encode(this.raw);
- }
-
- toBase64(): string {
- return encodeBase64(this.raw);
- }
-
- startSimpleParse() {
- return { sections: () => new MailSimpleParsedSections(this.raw) };
- }
-
- simpleFindAllAddresses(): string[] {
- const re = /,?\<?([a-z0-9_'+\-\.]+\@[a-z0-9_'+\-\.]+)\>?,?/gi;
- return [...this.raw.matchAll(re)].map((m) => m[1]);
- }
-}
-
-export type MailDeliverResultKind = "done" | "fail";
-
-export interface MailDeliverRecipientResult {
- kind: MailDeliverResultKind;
- message: string;
- cause?: unknown;
-}
-
-export class MailDeliverResult {
- message: string = "";
- recipients: Map<string, MailDeliverRecipientResult> = new Map();
-
- constructor(public mail: Mail) {}
-
- hasError(): boolean {
- return (
- this.recipients.size === 0 ||
- this.recipients.values().some((r) => r.kind !== "done")
- );
- }
-
- [Symbol.for("Deno.customInspect")]() {
- return [
- `message: ${this.message}`,
- ...this.recipients
- .entries()
- .map(
- ([recipient, result]) =>
- `${recipient} [${result.kind}]: ${result.message}`,
- ),
- ].join("\n");
- }
-}
-
-export class MailDeliverContext {
- readonly recipients: Set<string> = new Set();
- readonly result;
-
- constructor(public mail: Mail) {
- this.result = new MailDeliverResult(this.mail);
- }
-}
-
-export interface MailDeliverHook {
- callback(context: MailDeliverContext): Promise<void>;
-}
-
-export abstract class MailDeliverer {
- abstract readonly name: string;
- preHooks: MailDeliverHook[] = [];
- postHooks: MailDeliverHook[] = [];
-
- protected abstract doDeliver(
- mail: Mail,
- context: MailDeliverContext,
- ): Promise<void>;
-
- async deliverRaw(rawMail: string) {
- return await this.deliver({ mail: new Mail(rawMail) });
- }
-
- async deliver(options: {
- mail: Mail;
- recipients?: string[];
- }): Promise<MailDeliverResult> {
- console.info(`Begin to deliver mail via ${this.name}...`);
-
- const context = new MailDeliverContext(options.mail);
- options.recipients?.forEach((r) => context.recipients.add(r));
-
- for (const hook of this.preHooks) {
- await hook.callback(context);
- }
-
- await this.doDeliver(context.mail, context);
-
- for (const hook of this.postHooks) {
- await hook.callback(context);
- }
-
- console.info("Deliver result:");
- console.info(context.result);
-
- if (context.result.hasError()) {
- throw new Error("Mail failed to deliver.");
- }
-
- return context.result;
- }
-}
-
-export abstract class SyncMailDeliverer extends MailDeliverer {
- #last: Promise<void> = Promise.resolve();
-
- override async deliver(options: {
- mail: Mail;
- recipients?: string[];
- }): Promise<MailDeliverResult> {
- console.info(
- "The mail deliverer is sync. Wait for last delivering done...",
- );
- await this.#last;
- const result = super.deliver(options);
- this.#last = result.then(
- () => {},
- () => {},
- );
- return result;
- }
-}
-
-export class RecipientFromHeadersHook implements MailDeliverHook {
- constructor(public mailDomain: string) {}
-
- callback(context: MailDeliverContext) {
- if (context.recipients.size !== 0) {
- console.warn(
- "Recipients are already filled. Won't set them with ones in headers.",
- );
- } else {
- context.mail
- .startSimpleParse()
- .sections()
- .headers()
- .recipients({
- domain: this.mailDomain,
- })
- .forEach((r) => context.recipients.add(r));
-
- console.info(
- "Recipients found from mail headers: " +
- [...context.recipients].join(", "),
- );
- }
- return Promise.resolve();
- }
-}
-
-export class FallbackRecipientHook implements MailDeliverHook {
- constructor(public fallback: Set<string> = new Set()) {}
-
- callback(context: MailDeliverContext) {
- if (context.recipients.size === 0) {
- console.info(
- "No recipients, fill with fallback: " + [...this.fallback].join(", "),
- );
- this.fallback.forEach((a) => context.recipients.add(a));
- }
- return Promise.resolve();
- }
-}
-
-export class AliasRecipientMailHook implements MailDeliverHook {
- #aliasFile;
-
- constructor(aliasFile: string) {
- this.#aliasFile = aliasFile;
- }
-
- async #parseAliasFile(): Promise<Map<string, string>> {
- const result = new Map();
- if ((await Deno.stat(this.#aliasFile)).isFile) {
- console.info(`Found recipients alias file: ${this.#aliasFile}.`);
- const text = await Deno.readTextFile(this.#aliasFile);
- const csv = parse(text);
- for (const [real, ...aliases] of csv) {
- aliases.forEach((a) => result.set(a, real));
- }
- }
- return result;
- }
-
- async callback(context: MailDeliverContext) {
- const aliases = await this.#parseAliasFile();
- for (const recipient of [...context.recipients]) {
- const realRecipients = aliases.get(recipient);
- if (realRecipients != null) {
- console.info(
- `Recipient alias resolved: ${recipient} => ${realRecipients}.`,
- );
- context.recipients.delete(recipient);
- context.recipients.add(realRecipients);
- }
- }
- }
-}
diff --git a/deno/mail-relay/app.ts b/deno/mail/app.ts
index eeffc12..2a8c78a 100644
--- a/deno/mail-relay/app.ts
+++ b/deno/mail/app.ts
@@ -1,8 +1,6 @@
import { Hono } from "hono";
import { logger as honoLogger } from "hono/logger";
-import { LogFileProvider } from "@crupest/base/log";
-
import {
AliasRecipientMailHook,
FallbackRecipientHook,
@@ -13,20 +11,21 @@ import { DovecotMailDeliverer } from "./dovecot.ts";
import { DumbSmtpServer } from "./dumb-smtp-server.ts";
export function createInbound(
- logFileProvider: LogFileProvider,
{
fallback,
mailDomain,
aliasFile,
ldaPath,
+ doveadmPath,
}: {
fallback: string[];
mailDomain: string;
aliasFile: string;
ldaPath: string;
+ doveadmPath: string;
},
) {
- const deliverer = new DovecotMailDeliverer(logFileProvider, ldaPath);
+ const deliverer = new DovecotMailDeliverer(ldaPath, doveadmPath);
deliverer.preHooks.push(
new RecipientFromHeadersHook(mailDomain),
new FallbackRecipientHook(new Set(fallback)),
@@ -39,24 +38,24 @@ export function createHono(outbound: MailDeliverer, inbound: MailDeliverer) {
const hono = new Hono();
hono.onError((err, c) => {
- console.error("Hono handler throws an error.", err);
- return c.json({ msg: "Server error, check its log." }, 500);
+ console.error("Hono handler threw an uncaught error.", err);
+ return c.json({ message: "Server error, check its log." }, 500);
});
hono.use(honoLogger());
hono.post("/send/raw", async (context) => {
const body = await context.req.text();
if (body.trim().length === 0) {
- return context.json({ msg: "Can't send an empty mail." }, 400);
+ return context.json({ message: "Can't send an empty mail." }, 400);
} else {
const result = await outbound.deliverRaw(body);
return context.json({
- awsMessageId: result.awsMessageId,
+ newMessageId: result.newMessageId,
});
}
});
hono.post("/receive/raw", async (context) => {
await inbound.deliverRaw(await context.req.text());
- return context.json({ msg: "Done!" });
+ return context.json({ message: "Done!" });
});
return hono;
diff --git a/deno/mail-relay/aws/app.ts b/deno/mail/aws/app.ts
index 13db351..7e16488 100644
--- a/deno/mail-relay/aws/app.ts
+++ b/deno/mail/aws/app.ts
@@ -6,19 +6,16 @@ import { FetchHttpHandler } from "@smithy/fetch-http-handler";
// @ts-types="npm:@types/yargs"
import yargs from "yargs";
-import { LogFileProvider } from "@crupest/base/log";
import { ConfigDefinition, ConfigProvider } from "@crupest/base/config";
import { CronTask } from "@crupest/base/cron";
import { DbService } from "../db.ts";
-import { Mail } from "../mail.ts";
-import {
- AwsMailMessageIdRewriteHook,
- AwsMailMessageIdSaveHook,
-} from "./mail.ts";
+import { createHono, createInbound, createSmtp, sendMail } from "../app.ts";
+import { DovecotMailDeliverer } from "../dovecot.ts";
+import { MailDeliverer } from "../mail.ts";
+import { MessageIdRewriteHook, MessageIdSaveHook } from "../mail.ts";
import { AwsMailDeliverer } from "./deliver.ts";
-import { AwsMailFetcher, AwsS3MailConsumer } from "./fetch.ts";
-import { createInbound, createHono, sendMail, createSmtp } from "../app.ts";
+import { AwsMailFetcher, LiveMailNotFoundError } from "./fetch.ts";
const PREFIX = "crupest-mail-server";
const CONFIG_DEFINITIONS = {
@@ -47,6 +44,10 @@ const CONFIG_DEFINITIONS = {
description: "full path of lda executable",
default: "/dovecot/libexec/dovecot/dovecot-lda",
},
+ doveadmPath: {
+ description: "full path of doveadm executable",
+ default: "/dovecot/bin/doveadm",
+ },
inboundFallback: {
description: "comma separated addresses used as fallback recipients",
default: "",
@@ -96,14 +97,18 @@ function createAwsOptions({
function createOutbound(
awsOptions: ReturnType<typeof createAwsOptions>,
db: DbService,
+ local?: DovecotMailDeliverer,
) {
const deliverer = new AwsMailDeliverer(awsOptions);
deliverer.preHooks.push(
- new AwsMailMessageIdRewriteHook(db.messageIdToAws.bind(db)),
+ new MessageIdRewriteHook(db.messageIdToNew.bind(db)),
);
deliverer.postHooks.push(
- new AwsMailMessageIdSaveHook((original, aws) =>
- db.addMessageIdMap({ message_id: original, aws_message_id: aws }).then(),
+ new MessageIdSaveHook(
+ async (original, new_message_id, context) => {
+ await db.addMessageIdMap({ message_id: original, new_message_id });
+ void local?.saveNewSent(context.logTag, context.mail, original);
+ },
),
);
return deliverer;
@@ -114,15 +119,18 @@ function setupAwsHono(
options: {
path: string;
auth: string;
- callback: (s3Key: string, recipients?: string[]) => Promise<void>;
+ fetcher: AwsMailFetcher;
+ deliverer: MailDeliverer;
},
) {
+ let counter = 1;
+
hono.post(
`/${options.path}`,
async (ctx, next) => {
const auth = ctx.req.header("Authorization");
if (auth !== options.auth) {
- return ctx.json({ msg: "Bad auth!" }, 403);
+ return ctx.json({ message: "Bad auth!" }, 403);
}
await next();
},
@@ -134,19 +142,32 @@ function setupAwsHono(
}),
),
async (ctx) => {
+ const { fetcher, deliverer } = options;
const { key, recipients } = ctx.req.valid("json");
- await options.callback(key, recipients);
- return ctx.json({ msg: "Done!" });
+ try {
+ await fetcher.deliverLiveMail(
+ `[inbound ${counter++}]`,
+ key,
+ deliverer,
+ recipients,
+ );
+ } catch (e) {
+ if (e instanceof LiveMailNotFoundError) {
+ return ctx.json({ message: e.message });
+ }
+ throw e;
+ }
+ return ctx.json({ message: "Done!" });
},
);
}
-function createCron(fetcher: AwsMailFetcher, consumer: AwsS3MailConsumer) {
+function createCron(fetcher: AwsMailFetcher, deliverer: MailDeliverer) {
return new CronTask({
name: "live-mail-recycler",
interval: 6 * 3600 * 1000,
callback: () => {
- return fetcher.recycleLiveMails(consumer);
+ return fetcher.recycleLiveMails(deliverer);
},
startNow: true,
});
@@ -155,10 +176,7 @@ function createCron(fetcher: AwsMailFetcher, consumer: AwsS3MailConsumer) {
function createBaseServices() {
const config = new ConfigProvider(PREFIX, CONFIG_DEFINITIONS);
Deno.mkdirSync(config.get("dataPath"), { recursive: true });
- const logFileProvider = new LogFileProvider(
- join(config.get("dataPath"), "log"),
- );
- return { config, logFileProvider };
+ return { config };
}
function createAwsFetchOnlyServices() {
@@ -177,25 +195,25 @@ function createAwsFetchOnlyServices() {
function createAwsRecycleOnlyServices() {
const services = createAwsFetchOnlyServices();
- const { config, logFileProvider } = services;
+ const { config } = services;
- const inbound = createInbound(logFileProvider, {
+ const inbound = createInbound({
fallback: config.getList("inboundFallback"),
ldaPath: config.get("ldaPath"),
+ doveadmPath: config.get("doveadmPath"),
aliasFile: join(config.get("dataPath"), "aliases.csv"),
mailDomain: config.get("mailDomain"),
});
- const recycler = (rawMail: string, _: unknown): Promise<void> =>
- inbound.deliver({ mail: new Mail(rawMail) }).then();
- return { ...services, inbound, recycler };
+ return { ...services, inbound };
}
+
function createAwsServices() {
const services = createAwsRecycleOnlyServices();
- const { config, awsOptions } = services;
+ const { config, awsOptions, inbound } = services;
const dbService = new DbService(join(config.get("dataPath"), "db.sqlite"));
- const outbound = createOutbound(awsOptions, dbService);
+ const outbound = createOutbound(awsOptions, dbService, inbound);
return { ...services, dbService, outbound };
}
@@ -206,21 +224,23 @@ function createServerServices() {
const smtp = createSmtp(outbound);
const hono = createHono(outbound, inbound);
+
setupAwsHono(hono, {
path: config.get("awsInboundPath"),
auth: config.get("awsInboundKey"),
- callback: (s3Key, recipients) => {
- return fetcher.consumeS3Mail(s3Key, (rawMail, _) =>
- inbound.deliver({ mail: new Mail(rawMail), recipients }).then(),
- );
- },
+ fetcher,
+ deliverer: inbound,
});
return { ...services, smtp, hono };
}
-function serve(cron: boolean = false) {
- const { config, fetcher, recycler, smtp, hono } = createServerServices();
+async function serve(cron: boolean = false) {
+ const { config, fetcher, inbound, smtp, dbService, hono } =
+ createServerServices();
+
+ await dbService.migrate();
+
smtp.serve({
hostname: config.get("smtpHost"),
port: config.getInt("smtpPort"),
@@ -234,7 +254,7 @@ function serve(cron: boolean = false) {
);
if (cron) {
- createCron(fetcher, recycler);
+ createCron(fetcher, inbound);
}
}
@@ -248,13 +268,13 @@ async function listLives() {
}
async function recycleLives() {
- const { fetcher, recycler } = createAwsRecycleOnlyServices();
- await fetcher.recycleLiveMails(recycler);
+ const { fetcher, inbound } = createAwsRecycleOnlyServices();
+ await fetcher.recycleLiveMails(inbound);
}
if (import.meta.main) {
await yargs(Deno.args)
- .scriptName("mail-relay")
+ .scriptName("mail")
.command({
command: "sendmail",
describe: "send mail via this server's endpoint",
diff --git a/deno/mail-relay/aws/deliver.ts b/deno/mail/aws/deliver.ts
index a002eda..37a871d 100644
--- a/deno/mail-relay/aws/deliver.ts
+++ b/deno/mail/aws/deliver.ts
@@ -4,21 +4,15 @@ import {
SESv2ClientConfig,
} from "@aws-sdk/client-sesv2";
-import { Mail, MailDeliverContext, SyncMailDeliverer } from "../mail.ts";
+import { Mail, MailDeliverContext, MailDeliverer } from "../mail.ts";
-declare module "../mail.ts" {
- interface MailDeliverResult {
- awsMessageId?: string;
- }
-}
-
-export class AwsMailDeliverer extends SyncMailDeliverer {
+export class AwsMailDeliverer extends MailDeliverer {
readonly name = "aws";
readonly #aws;
readonly #ses;
constructor(aws: SESv2ClientConfig) {
- super();
+ super(true);
this.#aws = aws;
this.#ses = new SESv2Client(aws);
}
@@ -27,8 +21,6 @@ export class AwsMailDeliverer extends SyncMailDeliverer {
mail: Mail,
context: MailDeliverContext,
): Promise<void> {
- console.info("Begin to call aws send-email api...");
-
try {
const sendCommand = new SendEmailCommand({
Content: {
@@ -36,23 +28,28 @@ export class AwsMailDeliverer extends SyncMailDeliverer {
},
});
+ console.info(context.logTag, "Calling aws send-email api...");
const res = await this.#ses.send(sendCommand);
if (res.MessageId == null) {
- console.warn("Aws send-email returns no message id.");
+ console.warn(
+ context.logTag,
+ "AWS send-email returned null message id.",
+ );
} else {
- context.result.awsMessageId = `${res.MessageId}@${
- this.#aws.region
- }.amazonses.com`;
+ context.result.newMessageId =
+ `${res.MessageId}@${this.#aws.region}.amazonses.com`;
}
+ context.result.messageForSmtp =
+ `AWS Message ID: ${context.result.newMessageId}`;
context.result.recipients.set("*", {
- kind: "done",
- message: `Successfully called aws send-email, message id ${context.result.awsMessageId}.`,
+ kind: "success",
+ message: `Succeeded to call aws send-email api.`,
});
} catch (cause) {
context.result.recipients.set("*", {
- kind: "fail",
- message: "An error was thrown when calling aws send-email." + cause,
+ kind: "failure",
+ message: "A JS error was thrown when calling aws send-email." + cause,
cause,
});
}
diff --git a/deno/mail-relay/aws/fetch.ts b/deno/mail/aws/fetch.ts
index 68e02e6..2154972 100644
--- a/deno/mail-relay/aws/fetch.ts
+++ b/deno/mail/aws/fetch.ts
@@ -3,13 +3,17 @@ import {
DeleteObjectCommand,
GetObjectCommand,
ListObjectsV2Command,
+ NoSuchKey,
S3Client,
S3ClientConfig,
} from "@aws-sdk/client-s3";
-import { toFileNameString } from "@crupest/base";
+import { DateUtils } from "@crupest/base";
import { Mail } from "../mail.ts";
+import { MailDeliverer } from "../mail.ts";
+
+export class LiveMailNotFoundError extends Error {}
async function s3MoveObject(
client: S3Client,
@@ -33,11 +37,6 @@ async function s3MoveObject(
const AWS_SES_S3_SETUP_TAG = "AMAZON_SES_SETUP_NOTIFICATION";
-export type AwsS3MailConsumer = (
- rawMail: string,
- s3Key: string,
-) => Promise<void>;
-
export class AwsMailFetcher {
readonly #livePrefix = "mail/live/";
readonly #archivePrefix = "mail/archive/";
@@ -50,8 +49,6 @@ export class AwsMailFetcher {
}
async listLiveMails(): Promise<string[]> {
- console.info("Begin to retrieve live mails.");
-
const listCommand = new ListObjectsV2Command({
Bucket: this.#bucket,
Prefix: this.#livePrefix,
@@ -59,14 +56,14 @@ export class AwsMailFetcher {
const res = await this.#s3.send(listCommand);
if (res.Contents == null) {
- console.warn("Listing live mails in S3 returns null Content.");
+ console.warn("S3 API returned null Content.");
return [];
}
const result: string[] = [];
for (const object of res.Contents) {
if (object.Key == null) {
- console.warn("Listing live mails in S3 returns an object with no Key.");
+ console.warn("S3 API returned null Key.");
continue;
}
@@ -77,50 +74,63 @@ export class AwsMailFetcher {
return result;
}
- async consumeS3Mail(s3Key: string, consumer: AwsS3MailConsumer) {
- console.info(`Begin to consume s3 mail ${s3Key} ...`);
-
- console.info(`Fetching s3 mail ${s3Key}...`);
+ async deliverLiveMail(
+ logTag: string,
+ s3Key: string,
+ deliverer: MailDeliverer,
+ recipients?: string[],
+ ) {
+ console.info(logTag, `Fetching live mail ${s3Key}...`);
const mailPath = `${this.#livePrefix}${s3Key}`;
const command = new GetObjectCommand({
Bucket: this.#bucket,
Key: mailPath,
});
- const res = await this.#s3.send(command);
- if (res.Body == null) {
- throw new Error("S3 mail returns a null body.");
- }
+ let rawMail;
- const rawMail = await res.Body.transformToString();
- console.info(`Done fetching s3 mail ${s3Key}.`);
+ try {
+ const res = await this.#s3.send(command);
+ if (res.Body == null) {
+ throw new Error("S3 API returns a null body.");
+ }
+ rawMail = await res.Body.transformToString();
+ } catch (cause) {
+ if (cause instanceof NoSuchKey) {
+ const message =
+ `Live mail ${s3Key} is not found. Perhaps already delivered?`;
+ console.error(message, cause);
+ throw new LiveMailNotFoundError(message);
+ }
+ throw cause;
+ }
- console.info(`Calling consumer...`);
- await consumer(rawMail, s3Key);
- console.info(`Done consuming s3 mail ${s3Key}.`);
+ const mail = new Mail(rawMail);
+ await deliverer.deliver({ mail, recipients });
- const date = new Mail(rawMail)
- .startSimpleParse()
- .sections()
- .headers()
- .date();
- const dateString =
- date != null ? toFileNameString(date, true) : "invalid-date";
+ const { date } = new Mail(rawMail).parsed;
+ const dateString = date != null
+ ? DateUtils.toFileNameString(date, true)
+ : "invalid-date";
const newPath = `${this.#archivePrefix}${dateString}/${s3Key}`;
- console.info(`Archiving s3 mail ${s3Key} to ${newPath}...`);
+ console.info(logTag, `Archiving live mail ${s3Key} to ${newPath}...`);
await s3MoveObject(this.#s3, this.#bucket, mailPath, newPath);
- console.info(`Done archiving s3 mail ${s3Key}.`);
- console.info(`Done consuming s3 mail ${s3Key}.`);
+ console.info(logTag, `Done deliver live mail ${s3Key}.`);
}
- async recycleLiveMails(consumer: AwsS3MailConsumer) {
+ async recycleLiveMails(deliverer: MailDeliverer) {
console.info("Begin to recycle live mails...");
const mails = await this.listLiveMails();
console.info(`Found ${mails.length} live mails`);
+ let counter = 1;
for (const s3Key of mails) {
- await this.consumeS3Mail(s3Key, consumer);
+ await this.deliverLiveMail(
+ `[${counter++}/${mails.length}]`,
+ s3Key,
+ deliverer,
+ );
}
}
}
diff --git a/deno/mail-relay/db.test.ts b/deno/mail/db.test.ts
index 60035c4..8a9ad27 100644
--- a/deno/mail-relay/db.test.ts
+++ b/deno/mail/db.test.ts
@@ -6,17 +6,17 @@ import { DbService } from "./db.ts";
describe("DbService", () => {
const mockRow = {
message_id: "mock-message-id@mock.mock",
- aws_message_id: "mock-aws-message-id@mock.mock",
+ new_message_id: "mock-new-message-id@mock.mock",
};
it("works", async () => {
const db = new DbService(":memory:");
await db.migrate();
await db.addMessageIdMap(mockRow);
- expect(await db.messageIdToAws(mockRow.message_id)).toBe(
- mockRow.aws_message_id,
+ expect(await db.messageIdToNew(mockRow.message_id)).toBe(
+ mockRow.new_message_id,
);
- expect(await db.messageIdFromAws(mockRow.aws_message_id)).toBe(
+ expect(await db.messageIdFromNew(mockRow.new_message_id)).toBe(
mockRow.message_id,
);
});
diff --git a/deno/mail-relay/db.ts b/deno/mail/db.ts
index 062700b..e41f762 100644
--- a/deno/mail-relay/db.ts
+++ b/deno/mail/db.ts
@@ -53,14 +53,14 @@ class SqliteDatabaseAdapter implements SqliteDatabase {
export class DbError extends Error {}
-interface AwsMessageIdMapTable {
+interface MessageIdMapTable {
id: Generated<number>;
message_id: string;
- aws_message_id: string;
+ new_message_id: string;
}
interface Database {
- aws_message_id_map: AwsMessageIdMapTable;
+ message_id_map: MessageIdMapTable;
}
const migrations: Record<string, Migration> = {
@@ -68,16 +68,16 @@ const migrations: Record<string, Migration> = {
// deno-lint-ignore no-explicit-any
async up(db: Kysely<any>): Promise<void> {
await db.schema
- .createTable("aws_message_id_map")
+ .createTable("message_id_map")
.addColumn("id", "integer", (col) => col.primaryKey().autoIncrement())
.addColumn("message_id", "text", (col) => col.notNull().unique())
- .addColumn("aws_message_id", "text", (col) => col.notNull().unique())
+ .addColumn("new_message_id", "text", (col) => col.notNull().unique())
.execute();
- for (const column of ["message_id", "aws_message_id"]) {
+ for (const column of ["message_id", "new_message_id"]) {
await db.schema
- .createIndex(`aws_message_id_map_${column}`)
- .on("aws_message_id_map")
+ .createIndex(`message_id_map_${column}`)
+ .on("message_id_map")
.column(column)
.execute();
}
@@ -85,7 +85,7 @@ const migrations: Record<string, Migration> = {
// deno-lint-ignore no-explicit-any
async down(db: Kysely<any>): Promise<void> {
- await db.schema.dropTable("aws_message_id_map").execute();
+ await db.schema.dropTable("message_id_map").execute();
},
},
};
@@ -117,28 +117,28 @@ export class DbService {
}
async addMessageIdMap(
- mail: Insertable<AwsMessageIdMapTable>,
+ mail: Insertable<MessageIdMapTable>,
): Promise<number> {
const inserted = await this.#kysely
- .insertInto("aws_message_id_map")
+ .insertInto("message_id_map")
.values(mail)
.executeTakeFirstOrThrow();
return Number(inserted.insertId!);
}
- async messageIdToAws(messageId: string): Promise<string | null> {
+ async messageIdToNew(messageId: string): Promise<string | null> {
const row = await this.#kysely
- .selectFrom("aws_message_id_map")
+ .selectFrom("message_id_map")
.where("message_id", "=", messageId)
- .select("aws_message_id")
+ .select("new_message_id")
.executeTakeFirst();
- return row?.aws_message_id ?? null;
+ return row?.new_message_id ?? null;
}
- async messageIdFromAws(awsMessageId: string): Promise<string | null> {
+ async messageIdFromNew(newMessageId: string): Promise<string | null> {
const row = await this.#kysely
- .selectFrom("aws_message_id_map")
- .where("aws_message_id", "=", awsMessageId)
+ .selectFrom("message_id_map")
+ .where("new_message_id", "=", newMessageId)
.select("message_id")
.executeTakeFirst();
return row?.message_id ?? null;
diff --git a/deno/mail-relay/deno.json b/deno/mail/deno.json
index 9105747..86a8999 100644
--- a/deno/mail-relay/deno.json
+++ b/deno/mail/deno.json
@@ -2,7 +2,7 @@
"version": "0.1.0",
"tasks": {
"run": "deno run -A aws/app.ts",
- "compile": "deno compile -o out/crupest-relay -A aws/app.ts"
+ "compile": "deno compile -o out/crupest-mail -A aws/app.ts"
},
"imports": {
"@aws-sdk/client-s3": "npm:@aws-sdk/client-s3@^3.821.0",
diff --git a/deno/mail/dovecot.ts b/deno/mail/dovecot.ts
new file mode 100644
index 0000000..c0d56a2
--- /dev/null
+++ b/deno/mail/dovecot.ts
@@ -0,0 +1,219 @@
+import { Mail, MailDeliverContext, MailDeliverer } from "./mail.ts";
+
+// https://doc.dovecot.org/main/core/man/dovecot-lda.1.html
+const ldaExitCodeMessageMap = new Map<number, string>();
+ldaExitCodeMessageMap.set(67, "recipient user not known");
+ldaExitCodeMessageMap.set(75, "temporary error");
+
+type CommandResult = {
+ kind: "exit";
+ status: Deno.CommandStatus;
+ logMessage: string;
+} | { kind: "throw"; cause: unknown; logMessage: string };
+
+async function runCommand(
+ bin: string,
+ options: {
+ logTag: string;
+ args: string[];
+ stdin?: Uint8Array;
+ suppressStartLog?: boolean;
+ suppressResultLog?: boolean;
+ errorCodeMessageMap?: Map<number, string>;
+ },
+): Promise<CommandResult> {
+ const { logTag, args, stdin, suppressResultLog, errorCodeMessageMap } =
+ options;
+
+ if (options.suppressResultLog !== true) {
+ console.info(logTag, `Run external command ${bin} ${args.join(" ")}`);
+ }
+
+ try {
+ // Create and spawn process.
+ const command = new Deno.Command(bin, {
+ args,
+ stdin: stdin == null ? "null" : "piped",
+ });
+ const process = command.spawn();
+
+ // Write stdin if any.
+ if (stdin != null) {
+ const writer = process.stdin.getWriter();
+ await writer.write(stdin);
+ writer.close();
+ }
+
+ // Wait for process to exit.
+ const status = await process.status;
+
+ // Build log message string.
+ let message = `External command exited with code ${status.code}`;
+ if (status.signal != null) message += ` (signal: ${status.signal})`;
+ if (errorCodeMessageMap != null && errorCodeMessageMap.has(status.code)) {
+ message += `, ${errorCodeMessageMap.get(status.code)}`;
+ }
+ message += ".";
+ if (suppressResultLog !== true) console.log(logTag, message);
+
+ // Return result.
+ return {
+ kind: "exit",
+ status,
+ logMessage: message,
+ };
+ } catch (cause) {
+ const message = `A JS error was thrown when invoking external command:`;
+ if (suppressResultLog !== true) console.log(logTag, message);
+ return { kind: "throw", cause, logMessage: message + " " + cause };
+ }
+}
+
+export class DovecotMailDeliverer extends MailDeliverer {
+ readonly name = "dovecot";
+ readonly #ldaPath;
+ readonly #doveadmPath;
+
+ constructor(
+ ldaPath: string,
+ doveadmPath: string,
+ ) {
+ super(false);
+ this.#ldaPath = ldaPath;
+ this.#doveadmPath = doveadmPath;
+ }
+
+ protected override async doDeliver(
+ mail: Mail,
+ context: MailDeliverContext,
+ ): Promise<void> {
+ const utf8Bytes = mail.toUtf8Bytes();
+
+ const recipients = [...context.recipients];
+
+ if (recipients.length === 0) {
+ throw new Error(
+ "Failed to deliver to dovecot, no recipients are specified.",
+ );
+ }
+
+ for (const recipient of recipients) {
+ const result = await runCommand(
+ this.#ldaPath,
+ {
+ logTag: context.logTag,
+ args: ["-d", recipient],
+ stdin: utf8Bytes,
+ suppressResultLog: true,
+ errorCodeMessageMap: ldaExitCodeMessageMap,
+ },
+ );
+
+ if (result.kind === "exit" && result.status.success) {
+ context.result.recipients.set(recipient, {
+ kind: "success",
+ message: result.logMessage,
+ });
+ } else {
+ context.result.recipients.set(recipient, {
+ kind: "failure",
+ message: result.logMessage,
+ });
+ }
+ }
+ }
+
+ #queryArgs(mailbox: string, messageId: string) {
+ return ["mailbox", mailbox, "header", "Message-ID", `<${messageId}>`];
+ }
+
+ async #deleteMail(
+ logTag: string,
+ user: string,
+ mailbox: string,
+ messageId: string,
+ noLog?: boolean,
+ ): Promise<void> {
+ await runCommand(this.#doveadmPath, {
+ logTag,
+ args: ["expunge", "-u", user, ...this.#queryArgs(mailbox, messageId)],
+ suppressStartLog: noLog,
+ suppressResultLog: noLog,
+ });
+ }
+
+ async #saveMail(
+ logTag: string,
+ user: string,
+ mailbox: string,
+ mail: Uint8Array,
+ ) {
+ await runCommand(this.#doveadmPath, {
+ logTag,
+ args: ["save", "-u", user, "-m", mailbox],
+ stdin: mail,
+ });
+ }
+
+ async #markAsRead(
+ logTag: string,
+ user: string,
+ mailbox: string,
+ messageId: string,
+ ) {
+ await runCommand(this.#doveadmPath, {
+ logTag,
+ args: [
+ "flags",
+ "add",
+ "-u",
+ user,
+ "\\Seen",
+ ...this.#queryArgs(mailbox, messageId),
+ ],
+ });
+ }
+
+ async saveNewSent(logTag: string, mail: Mail, messageIdToDelete: string) {
+ console.info(logTag, "Save sent mail and delete ones with old message id.");
+
+ // Try to get from and recipients from headers.
+ const { messageId, from, recipients } = mail.parsed;
+
+ if (from == null) {
+ console.warn(
+ logTag,
+ "Failed to get sender (from) in headers, skip saving.",
+ );
+ return;
+ }
+
+ if (recipients.includes(from)) {
+ // So the mail should lie in the Inbox.
+ console.info(
+ logTag,
+ "One recipient of the mail is the sender itself, skip saving.",
+ );
+ return;
+ }
+
+ await this.#saveMail(logTag, from, "Sent", mail.toUtf8Bytes());
+ if (messageId != null) {
+ await this.#markAsRead(logTag, from, "Sent", messageId);
+ } else {
+ console.warn(
+ "Message id of the mail is not found, skip marking as read.",
+ );
+ }
+
+ console.info(
+ logTag,
+ "Schedule deletion of old mails (no logging) at 5,15,30,60 seconds later.",
+ );
+ [5, 15, 30, 60].forEach((seconds) =>
+ setTimeout(() => {
+ void this.#deleteMail(logTag, from, "Sent", messageIdToDelete, true);
+ }, 1000 * seconds)
+ );
+ }
+}
diff --git a/deno/mail-relay/dumb-smtp-server.ts b/deno/mail/dumb-smtp-server.ts
index ac7069c..c3ebf5d 100644
--- a/deno/mail-relay/dumb-smtp-server.ts
+++ b/deno/mail/dumb-smtp-server.ts
@@ -12,31 +12,30 @@ function createResponses(host: string, port: number | string) {
RCPT: "250 2.1.5 Recipient OK",
DATA: "354 Start mail input; end with <CRLF>.<CRLF>",
QUIT: `211 2.0.0 ${serverName} closing connection`,
+ ACTIVE_CLOSE: "421 4.7.0 Please open a new connection to send more emails",
INVALID: "500 5.5.1 Error: command not recognized",
} as const;
}
-const LOG_TAG = "[dumb-smtp]";
-
export class DumbSmtpServer {
#deliverer;
- #responses: ReturnType<typeof createResponses> = createResponses(
- "invalid",
- "invalid",
- );
constructor(deliverer: MailDeliverer) {
this.#deliverer = deliverer;
}
- async #handleConnection(conn: Deno.Conn) {
+ async #handleConnection(
+ logTag: string,
+ conn: Deno.Conn,
+ responses: ReturnType<typeof createResponses>,
+ ) {
using disposeStack = new DisposableStack();
disposeStack.defer(() => {
- console.info(LOG_TAG, "Close session's tcp connection.");
+ console.info(logTag, "Close tcp connection.");
conn.close();
});
- console.info(LOG_TAG, "New session's tcp connection established.");
+ console.info(logTag, "New tcp connection established.");
const writer = conn.writable.getWriter();
disposeStack.defer(() => writer.releaseLock());
@@ -46,14 +45,14 @@ export class DumbSmtpServer {
const [decoder, encoder] = [new TextDecoder(), new TextEncoder()];
const decode = (data: Uint8Array) => decoder.decode(data);
const send = async (s: string) => {
- console.info(LOG_TAG, "Send line: " + s);
+ console.info(logTag, "Send line:", s);
await writer.write(encoder.encode(s + CRLF));
};
let buffer: string = "";
let rawMail: string | null = null;
- await send(this.#responses["READY"]);
+ await send(responses["READY"]);
while (true) {
const { value, done } = await reader.read();
@@ -69,39 +68,37 @@ export class DumbSmtpServer {
buffer = buffer.slice(eolPos + CRLF.length);
if (rawMail == null) {
- console.info(LOG_TAG, "Received line: " + line);
+ console.info(logTag, "Received line:", line);
const upperLine = line.toUpperCase();
if (upperLine.startsWith("EHLO") || upperLine.startsWith("HELO")) {
- await send(this.#responses["EHLO"]);
+ await send(responses["EHLO"]);
} else if (upperLine.startsWith("MAIL FROM:")) {
- await send(this.#responses["MAIL"]);
+ await send(responses["MAIL"]);
} else if (upperLine.startsWith("RCPT TO:")) {
- await send(this.#responses["RCPT"]);
+ await send(responses["RCPT"]);
} else if (upperLine === "DATA") {
- await send(this.#responses["DATA"]);
- console.info(LOG_TAG, "Begin to receive mail data...");
+ await send(responses["DATA"]);
+ console.info(logTag, "Begin to receive mail data...");
rawMail = "";
} else if (upperLine === "QUIT") {
- await send(this.#responses["QUIT"]);
+ await send(responses["QUIT"]);
return;
} else {
- console.warn(LOG_TAG, "Unrecognized command from client: " + line);
- await send(this.#responses["INVALID"]);
+ await send(responses["INVALID"]);
return;
}
} else {
if (line === ".") {
try {
- console.info(LOG_TAG, "Mail data Received, begin to relay...");
- const { message } = await this.#deliverer.deliverRaw(rawMail);
- await send(`250 2.6.0 ${message}`);
+ console.info(logTag, "Mail data received, begin to relay...");
+ const result = await this.#deliverer.deliverRaw(rawMail);
+ await send(`250 2.6.0 ${result.generateMessageForSmtp()}`);
rawMail = null;
- console.info(LOG_TAG, "Relay succeeded.");
} catch (err) {
- console.error(LOG_TAG, "Relay failed.", err);
+ console.error(logTag, "Relay failed.", err);
await send("554 5.3.0 Error: check server log");
- return;
}
+ await send(responses["ACTIVE_CLOSE"]);
} else {
const dataLine = line.startsWith("..") ? line.slice(1) : line;
rawMail += dataLine + CRLF;
@@ -113,17 +110,19 @@ export class DumbSmtpServer {
async serve(options: { hostname: string; port: number }) {
const listener = Deno.listen(options);
- this.#responses = createResponses(options.hostname, options.port);
+ const responses = createResponses(options.hostname, options.port);
console.info(
- LOG_TAG,
- `Dumb SMTP server starts to listen on ${this.#responses.serverName}.`,
+ `Dumb SMTP server starts to listen on ${responses.serverName}.`,
);
+ let counter = 1;
+
for await (const conn of listener) {
+ const logTag = `[outbound ${counter++}]`;
try {
- await this.#handleConnection(conn);
+ await this.#handleConnection(logTag, conn, responses);
} catch (cause) {
- console.error(LOG_TAG, "Tcp connection throws an error.", cause);
+ console.error(logTag, "A JS error was thrown by handler:", cause);
}
}
}
diff --git a/deno/mail/mail-parsing.ts b/deno/mail/mail-parsing.ts
new file mode 100644
index 0000000..8e9697d
--- /dev/null
+++ b/deno/mail/mail-parsing.ts
@@ -0,0 +1,144 @@
+import emailAddresses from "email-addresses";
+
+class MailParsingError extends Error {}
+
+function parseHeaderSection(section: string) {
+ const headers = [] as [key: string, value: string][];
+
+ let field: string | null = null;
+ let lineNumber = 1;
+
+ const handleField = () => {
+ if (field == null) return;
+ const sepPos = field.indexOf(":");
+ if (sepPos === -1) {
+ throw new MailParsingError(
+ `Expect ':' in the header field line: ${field}`,
+ );
+ }
+ headers.push([field.slice(0, sepPos).trim(), field.slice(sepPos + 1)]);
+ field = null;
+ };
+
+ for (const line of section.trimEnd().split(/\r?\n|\r/)) {
+ if (line.match(/^\s/)) {
+ if (field == null) {
+ throw new MailParsingError("Header section starts with a space.");
+ }
+ field += line;
+ } else {
+ handleField();
+ field = line;
+ }
+ lineNumber += 1;
+ }
+
+ handleField();
+
+ return headers;
+}
+
+function findFirst(fields: readonly [string, string][], key: string) {
+ for (const [k, v] of fields) {
+ if (key.toLowerCase() === k.toLowerCase()) return v;
+ }
+ return undefined;
+}
+
+function findMessageId(fields: readonly [string, string][]) {
+ const messageIdField = findFirst(fields, "message-id");
+ if (messageIdField == null) return undefined;
+
+ const match = messageIdField.match(/\<(.*?)\>/);
+ if (match != null) {
+ return match[1];
+ } else {
+ console.warn(`Invalid syntax in header 'message-id': ${messageIdField}`);
+ return undefined;
+ }
+}
+
+function findDate(fields: readonly [string, string][]) {
+ const dateField = findFirst(fields, "date");
+ if (dateField == null) return undefined;
+
+ const date = new Date(dateField);
+ if (isNaN(date.getTime())) {
+ console.warn(`Invalid date string in header 'date': ${dateField}`);
+ return undefined;
+ }
+ return date;
+}
+
+function findFrom(fields: readonly [string, string][]) {
+ const fromField = findFirst(fields, "from");
+ if (fromField == null) return undefined;
+
+ const addr = emailAddresses.parseOneAddress(fromField);
+ return addr?.type === "mailbox" ? addr.address : undefined;
+}
+
+function findRecipients(fields: readonly [string, string][]) {
+ const headers = ["to", "cc", "bcc", "x-original-to"];
+ const recipients = new Set<string>();
+ for (const [key, value] of fields) {
+ if (headers.includes(key.toLowerCase())) {
+ emailAddresses
+ .parseAddressList(value)
+ ?.flatMap((a) => (a.type === "mailbox" ? a : a.addresses))
+ ?.forEach(({ address }) => recipients.add(address));
+ }
+ }
+ return recipients;
+}
+
+function parseSections(raw: string) {
+ const twoEolMatch = raw.match(/(\r?\n)(\r?\n)/);
+ if (twoEolMatch == null) {
+ throw new MailParsingError(
+ "No header/body section separator (2 successive EOLs) found.",
+ );
+ }
+
+ const [eol, sep] = [twoEolMatch[1], twoEolMatch[2]];
+
+ if (eol !== sep) {
+ console.warn("Different EOLs (\\r\\n, \\n) found.");
+ }
+
+ return {
+ header: raw.slice(0, twoEolMatch.index!),
+ body: raw.slice(twoEolMatch.index! + eol.length + sep.length),
+ eol,
+ sep,
+ };
+}
+
+export type ParsedMail = Readonly<{
+ header: string;
+ body: string;
+ sep: string;
+ eol: string;
+ headers: readonly [string, string][];
+ messageId: string | undefined;
+ date: Date | undefined;
+ from: string | undefined;
+ recipients: readonly string[];
+}>;
+
+export function simpleParseMail(raw: string): ParsedMail {
+ const sections = Object.freeze(parseSections(raw));
+ const headers = Object.freeze(parseHeaderSection(sections.header));
+ const messageId = findMessageId(headers);
+ const date = findDate(headers);
+ const from = findFrom(headers);
+ const recipients = Object.freeze([...findRecipients(headers)]);
+ return Object.freeze({
+ ...sections,
+ headers,
+ messageId,
+ date,
+ from,
+ recipients,
+ });
+}
diff --git a/deno/mail-relay/mail.test.ts b/deno/mail/mail.test.ts
index 6abb7d7..a8204be 100644
--- a/deno/mail-relay/mail.test.ts
+++ b/deno/mail/mail.test.ts
@@ -51,7 +51,7 @@ const mockToAddresses = [
describe("Mail", () => {
it("simple parse", () => {
- const parsed = new Mail(mockMailStr).startSimpleParse().sections();
+ const { parsed } = new Mail(mockMailStr);
expect(parsed.header).toEqual(mockHeaderStr);
expect(parsed.body).toEqual(mockBodyStr);
expect(parsed.sep).toBe("\n");
@@ -59,37 +59,29 @@ describe("Mail", () => {
});
it("simple parse crlf", () => {
- const parsed = new Mail(mockCrlfMailStr).startSimpleParse().sections();
+ const { parsed } = new Mail(mockCrlfMailStr);
expect(parsed.sep).toBe("\r\n");
expect(parsed.eol).toBe("\r\n");
});
it("simple parse date", () => {
expect(
- new Mail(mockMailStr).startSimpleParse().sections().headers().date(),
+ new Mail(mockMailStr).parsed.date,
).toEqual(new Date(mockDate));
});
it("simple parse headers", () => {
expect(
- new Mail(mockMailStr).startSimpleParse().sections().headers().fields,
+ new Mail(mockMailStr).parsed.headers,
).toEqual(mockHeaders.map((h) => [h[0], " " + h[1].replaceAll("\n", "")]));
});
it("parse recipients", () => {
const mail = new Mail(mockMailStr);
- expect([
- ...mail.startSimpleParse().sections().headers().recipients(),
- ]).toEqual([...mockToAddresses, mockCcAddress]);
- expect([
- ...mail.startSimpleParse().sections().headers().recipients({
- domain: "example.com",
- }),
- ]).toEqual(
- [...mockToAddresses, mockCcAddress].filter((a) =>
- a.endsWith("example.com"),
- ),
- );
+ expect([...mail.parsed.recipients]).toEqual([
+ ...mockToAddresses,
+ mockCcAddress,
+ ]);
});
it("find all addresses", () => {
@@ -113,11 +105,14 @@ describe("MailDeliverer", () => {
class MockMailDeliverer extends MailDeliverer {
name = "mock";
override doDeliver = fn((_: Mail, ctx: MailDeliverContext) => {
- ctx.result.recipients.set("*", { kind: "done", message: "success" });
+ ctx.result.recipients.set("*", {
+ kind: "success",
+ message: "success message",
+ });
return Promise.resolve();
}) as MailDeliverer["doDeliver"];
}
- const mockDeliverer = new MockMailDeliverer();
+ const mockDeliverer = new MockMailDeliverer(false);
it("deliver success", async () => {
await mockDeliverer.deliverRaw(mockMailStr);
diff --git a/deno/mail/mail.ts b/deno/mail/mail.ts
new file mode 100644
index 0000000..b88ce2b
--- /dev/null
+++ b/deno/mail/mail.ts
@@ -0,0 +1,304 @@
+import { encodeBase64 } from "@std/encoding/base64";
+import { parse } from "@std/csv/parse";
+
+import { StringUtils } from "@crupest/base";
+
+import { simpleParseMail } from "./mail-parsing.ts";
+
+export class Mail {
+ #raw;
+ #parsed;
+
+ constructor(raw: string) {
+ this.#raw = raw;
+ this.#parsed = simpleParseMail(raw);
+ }
+
+ get raw() {
+ return this.#raw;
+ }
+
+ set raw(value) {
+ this.#raw = value;
+ this.#parsed = simpleParseMail(value);
+ }
+
+ get parsed() {
+ return this.#parsed;
+ }
+
+ toUtf8Bytes(): Uint8Array {
+ const utf8Encoder = new TextEncoder();
+ return utf8Encoder.encode(this.raw);
+ }
+
+ toBase64(): string {
+ return encodeBase64(this.raw);
+ }
+
+ simpleFindAllAddresses(): string[] {
+ const re = /,?\<?([a-z0-9_'+\-\.]+\@[a-z0-9_'+\-\.]+)\>?,?/gi;
+ return [...this.raw.matchAll(re)].map((m) => m[1]);
+ }
+}
+
+export interface MailDeliverRecipientResult {
+ kind: "success" | "failure";
+ message?: string;
+ cause?: unknown;
+}
+
+export class MailDeliverResult {
+ message?: string;
+ messageForSmtp?: string;
+ newMessageId?: string;
+
+ recipients = new Map<string, MailDeliverRecipientResult>();
+ constructor(public mail: Mail) {}
+
+ get hasFailure() {
+ return this.recipients.values().some((v) => v.kind !== "success");
+ }
+
+ generateLogMessage(prefix: string) {
+ const lines = [];
+ if (this.message != null) lines.push(`${prefix} message: ${this.message}`);
+ if (this.messageForSmtp != null) {
+ lines.push(`${prefix} smtpMessage: ${this.messageForSmtp}`);
+ }
+ for (const [name, result] of this.recipients.entries()) {
+ const { kind, message } = result;
+ lines.push(`${prefix} (${name}): ${kind} ${message}`);
+ }
+ return lines.join("\n");
+ }
+
+ generateMessageForSmtp(): string {
+ if (this.messageForSmtp != null) return this.messageForSmtp;
+ return `2.0.0 OK${
+ StringUtils.prependNonEmpty(this.newMessageId)
+ } Message accepted for delivery`;
+ }
+}
+
+export class MailDeliverContext {
+ readonly recipients: Set<string> = new Set();
+ readonly result;
+
+ constructor(public logTag: string, public mail: Mail) {
+ this.result = new MailDeliverResult(this.mail);
+ }
+}
+
+export interface MailDeliverHook {
+ callback(context: MailDeliverContext): Promise<void>;
+}
+
+export abstract class MailDeliverer {
+ #counter = 1;
+ #last?: Promise<void>;
+
+ abstract name: string;
+ preHooks: MailDeliverHook[] = [];
+ postHooks: MailDeliverHook[] = [];
+
+ constructor(public sync: boolean) {}
+
+ protected abstract doDeliver(
+ mail: Mail,
+ context: MailDeliverContext,
+ ): Promise<void>;
+
+ async deliverRaw(rawMail: string) {
+ return await this.deliver({ mail: new Mail(rawMail) });
+ }
+
+ async #deliverCore(context: MailDeliverContext) {
+ for (const hook of this.preHooks) {
+ await hook.callback(context);
+ }
+
+ await this.doDeliver(context.mail, context);
+
+ for (const hook of this.postHooks) {
+ await hook.callback(context);
+ }
+ }
+
+ async deliver(options: {
+ mail: Mail;
+ recipients?: string[];
+ logTag?: string;
+ }): Promise<MailDeliverResult> {
+ const logTag = options.logTag ?? `[${this.name} ${this.#counter}]`;
+ this.#counter++;
+
+ if (this.#last != null) {
+ console.info(logTag, "Wait for last delivering done...");
+ await this.#last;
+ }
+
+ const context = new MailDeliverContext(
+ logTag,
+ options.mail,
+ );
+ options.recipients?.forEach((r) => context.recipients.add(r));
+
+ console.info(context.logTag, "Begin to deliver mail...");
+
+ const deliverPromise = this.#deliverCore(context);
+
+ if (this.sync) {
+ this.#last = deliverPromise.then(() => {}, () => {});
+ }
+
+ await deliverPromise;
+ this.#last = undefined;
+
+ console.info(context.logTag, "Deliver result:");
+ console.info(context.result.generateLogMessage(context.logTag));
+
+ if (context.result.hasFailure) {
+ throw new Error("Failed to deliver to some recipients.");
+ }
+
+ return context.result;
+ }
+}
+
+export class RecipientFromHeadersHook implements MailDeliverHook {
+ constructor(public mailDomain: string) {}
+
+ callback(context: MailDeliverContext) {
+ if (context.recipients.size !== 0) {
+ console.warn(
+ context.logTag,
+ "Recipients are already filled, skip inferring from headers.",
+ );
+ } else {
+ [...context.mail.parsed.recipients].filter((r) =>
+ r.endsWith("@" + this.mailDomain)
+ ).forEach((r) => context.recipients.add(r));
+
+ console.info(
+ context.logTag,
+ "Use recipients inferred from mail headers:",
+ [...context.recipients].join(", "),
+ );
+ }
+ return Promise.resolve();
+ }
+}
+
+export class FallbackRecipientHook implements MailDeliverHook {
+ constructor(public fallback: Set<string> = new Set()) {}
+
+ callback(context: MailDeliverContext) {
+ if (context.recipients.size === 0) {
+ console.info(
+ context.logTag,
+ "Use fallback recipients:" + [...this.fallback].join(", "),
+ );
+ this.fallback.forEach((a) => context.recipients.add(a));
+ }
+ return Promise.resolve();
+ }
+}
+
+export class AliasRecipientMailHook implements MailDeliverHook {
+ #aliasFile;
+
+ constructor(aliasFile: string) {
+ this.#aliasFile = aliasFile;
+ }
+
+ async #parseAliasFile(logTag: string): Promise<Map<string, string>> {
+ const result = new Map();
+ if ((await Deno.stat(this.#aliasFile)).isFile) {
+ const text = await Deno.readTextFile(this.#aliasFile);
+ const csv = parse(text);
+ for (const [real, ...aliases] of csv) {
+ aliases.forEach((a) => result.set(a, real));
+ }
+ } else {
+ console.warn(
+ logTag,
+ `Recipient alias file ${this.#aliasFile} is not found.`,
+ );
+ }
+ return result;
+ }
+
+ async callback(context: MailDeliverContext) {
+ const aliases = await this.#parseAliasFile(context.logTag);
+ for (const recipient of [...context.recipients]) {
+ const realRecipients = aliases.get(recipient);
+ if (realRecipients != null) {
+ console.info(
+ context.logTag,
+ `Recipient alias resolved: ${recipient} => ${realRecipients}.`,
+ );
+ context.recipients.delete(recipient);
+ context.recipients.add(realRecipients);
+ }
+ }
+ }
+}
+
+export class MessageIdRewriteHook implements MailDeliverHook {
+ readonly #lookup;
+
+ constructor(lookup: (origin: string) => Promise<string | null>) {
+ this.#lookup = lookup;
+ }
+
+ async callback(context: MailDeliverContext): Promise<void> {
+ const addresses = context.mail.simpleFindAllAddresses();
+ for (const address of addresses) {
+ const newMessageId = await this.#lookup(address);
+ if (newMessageId != null && newMessageId.length !== 0) {
+ console.info(
+ context.logTag,
+ `Rewrite address-line string in mail: ${address} => ${newMessageId}.`,
+ );
+ context.mail.raw = context.mail.raw.replaceAll(address, newMessageId);
+ }
+ }
+ }
+}
+
+export class MessageIdSaveHook implements MailDeliverHook {
+ readonly #record;
+
+ constructor(
+ record: (
+ original: string,
+ newMessageId: string,
+ context: MailDeliverContext,
+ ) => Promise<void>,
+ ) {
+ this.#record = record;
+ }
+
+ async callback(context: MailDeliverContext): Promise<void> {
+ const { messageId } = context.mail.parsed;
+ if (messageId == null) {
+ console.warn(
+ context.logTag,
+ "Original mail doesn't have message id, skip saving message id map.",
+ );
+ return;
+ }
+ if (context.result.newMessageId != null) {
+ console.info(
+ context.logTag,
+ `Save message id map: ${messageId} => ${context.result.newMessageId}.`,
+ );
+ context.mail.raw = context.mail.raw.replaceAll(
+ messageId,
+ context.result.newMessageId,
+ );
+ await this.#record(messageId, context.result.newMessageId, context);
+ }
+ }
+}
diff --git a/deno/tools/deno.json b/deno/tools/deno.json
index 1b2cf32..355046a 100644
--- a/deno/tools/deno.json
+++ b/deno/tools/deno.json
@@ -3,6 +3,6 @@
"tasks": {
},
"imports": {
- "mustache": "npm:mustache@^4.2.0",
+ "mustache": "npm:mustache@^4.2.0"
}
}
diff --git a/deno/tools/generate-geosite-rules.ts b/deno/tools/geosite.ts
index bfa53ba..3aabec2 100644
--- a/deno/tools/generate-geosite-rules.ts
+++ b/deno/tools/geosite.ts
@@ -1,4 +1,3 @@
-const PROXY_NAME = "node-select";
const ATTR = "cn";
const REPO_NAME = "domain-list-community";
const URL =
@@ -152,8 +151,10 @@ if (import.meta.main) {
const rules = extract(SITES, provider);
const [has, notHas] = toNewFormat(rules, ATTR);
- const hasFile = tmpDir + "/has-rule";
- const notHasFile = tmpDir + "/not-has-rule";
+ const resultDir = tmpDir + "/result";
+ Deno.mkdirSync(resultDir);
+ const hasFile = resultDir + "/has-rule";
+ const notHasFile = resultDir + "/not-has-rule";
console.log("Write result to: " + hasFile + " , " + notHasFile);
Deno.writeTextFileSync(hasFile, has);
Deno.writeTextFileSync(notHasFile, notHas);
diff --git a/deno/tools/main.ts b/deno/tools/main.ts
new file mode 100644
index 0000000..897350c
--- /dev/null
+++ b/deno/tools/main.ts
@@ -0,0 +1,14 @@
+import yargs, { DEMAND_COMMAND_MESSAGE } from "./yargs.ts";
+import vm from "./vm.ts";
+import service from "./service.ts";
+
+if (import.meta.main) {
+ await yargs(Deno.args)
+ .scriptName("crupest")
+ .command(vm)
+ .command(service)
+ .demandCommand(1, DEMAND_COMMAND_MESSAGE)
+ .help()
+ .strict()
+ .parse();
+}
diff --git a/deno/tools/manage-service.ts b/deno/tools/manage-service.ts
deleted file mode 100644
index 148f55a..0000000
--- a/deno/tools/manage-service.ts
+++ /dev/null
@@ -1,42 +0,0 @@
-import { join } from "@std/path";
-// @ts-types="npm:@types/yargs"
-import yargs from "yargs";
-
-import { TemplateDir } from "./template.ts";
-
-if (import.meta.main) {
- await yargs(Deno.args)
- .scriptName("manage-service")
- .option("project-dir", {
- type: "string",
- })
- .demandOption("project-dir")
- .command({
- command: "gen-tmpl",
- describe: "generate files for templates",
- builder: (builder) => {
- return builder
- .option("dry-run", {
- type: "boolean",
- default: true,
- })
- .strict();
- },
- handler: (argv) => {
- const { projectDir, dryRun } = argv;
- new TemplateDir(
- join(projectDir, "services/templates"),
- ).generateWithVariableFiles(
- [
- join(projectDir, "data/config"),
- join(projectDir, "services/config.template"),
- ],
- dryRun ? undefined : join(projectDir, "services/generated"),
- );
- },
- })
- .demandCommand(1, "One command must be specified.")
- .help()
- .strict()
- .parse();
-}
diff --git a/deno/tools/manage-vm.ts b/deno/tools/manage-vm.ts
deleted file mode 100644
index bb985ce..0000000
--- a/deno/tools/manage-vm.ts
+++ /dev/null
@@ -1,144 +0,0 @@
-import os from "node:os";
-import { join } from "@std/path";
-// @ts-types="npm:@types/yargs"
-import yargs from "yargs";
-
-type ArchAliasMap = { [name: string]: string[] };
-const arches = {
- x86_64: ["x86_64", "amd64"],
- i386: ["i386", "x86", "i686"],
-} as const satisfies ArchAliasMap;
-type Arch = keyof typeof arches;
-type GeneralArch = (typeof arches)[Arch][number];
-
-function normalizeArch(generalName: GeneralArch): Arch {
- for (const [name, aliases] of Object.entries(arches as ArchAliasMap)) {
- if (aliases.includes(generalName)) return name as Arch;
- }
- throw Error("Unknown architecture name.");
-}
-
-interface GeneralVmSetup {
- name?: string[];
- arch: GeneralArch;
- disk: string;
- sshForwardPort: number;
- kvm?: boolean;
-}
-
-interface VmSetup {
- arch: Arch;
- disk: string;
- sshForwardPort: number;
- kvm: boolean;
-}
-
-const MY_VMS: GeneralVmSetup[] = [
- {
- name: ["hurd", ...arches.i386.map((a) => `hurd-${a}`)],
- arch: "i386",
- disk: join(os.homedir(), "vms/hurd-i386.qcow2"),
- sshForwardPort: 3222,
- },
- {
- name: [...arches.x86_64.map((a) => `hurd-${a}`)],
- arch: "x86_64",
- disk: join(os.homedir(), "vms/hurd-x86_64.qcow2"),
- sshForwardPort: 3223,
- },
-];
-
-function normalizeVmSetup(generalSetup: GeneralVmSetup): VmSetup {
- const { arch, disk, sshForwardPort, kvm } = generalSetup;
- return {
- arch: normalizeArch(arch),
- disk,
- sshForwardPort,
- kvm: kvm ?? Deno.build.os === "linux",
- };
-}
-
-function resolveVmSetup(
- name: string,
- vms: GeneralVmSetup[],
-): VmSetup | undefined {
- const setup = vms.find((vm) => vm.name?.includes(name));
- return setup == null ? undefined : normalizeVmSetup(setup);
-}
-
-const qemuBinPrefix = "qemu-system" as const;
-
-const qemuBinSuffix = {
- x86_64: "x86_64",
- i386: "x86_64",
-} as const;
-
-function getQemuBin(arch: Arch): string {
- return `${qemuBinPrefix}-${qemuBinSuffix[arch]}`;
-}
-
-function getLinuxHostArgs(kvm: boolean): string[] {
- return kvm ? ["-enable-kvm"] : [];
-}
-
-function getMachineArgs(arch: Arch): string[] {
- const is64 = arch === "x86_64";
- const machineArgs = is64 ? ["-machine", "q35"] : [];
- const memory = is64 ? 8 : 4;
- return [...machineArgs, "-m", `${memory}G`];
-}
-
-function getNetworkArgs(sshForwardPort: number): string[] {
- return ["-net", "nic", "-net", `user,hostfwd=tcp::${sshForwardPort}-:22`];
-}
-
-function getDisplayArgs(): string[] {
- return ["-vga", "vmware"];
-}
-
-function getDiskArgs(disk: string): string[] {
- return ["-drive", `cache=writeback,file=${disk}`];
-}
-
-function createQemuArgs(setup: VmSetup): string[] {
- const { arch, disk, sshForwardPort } = setup;
- return [
- getQemuBin(arch),
- ...getLinuxHostArgs(setup.kvm),
- ...getMachineArgs(arch),
- ...getDisplayArgs(),
- ...getNetworkArgs(sshForwardPort),
- ...getDiskArgs(disk),
- ];
-}
-
-if (import.meta.main) {
- await yargs(Deno.args)
- .scriptName("manage-vm")
- .command({
- command: "gen <name>",
- describe: "generate cli command to run the vm",
- builder: (builder) => {
- return builder
- .positional("name", {
- describe: "name of the vm to run",
- type: "string",
- })
- .demandOption("name")
- .strict();
- },
- handler: (argv) => {
- const vm = resolveVmSetup(argv.name, MY_VMS);
- if (vm == null) {
- console.error(`No vm called ${argv.name} is found.`);
- Deno.exit(-1);
- }
- const cli = createQemuArgs(vm);
- console.log(`${cli.join(" ")}`);
- },
- })
- .demandCommand(1, "One command must be specified.")
- .help()
- .strict()
- .parse();
-}
diff --git a/deno/tools/service.ts b/deno/tools/service.ts
new file mode 100644
index 0000000..bd4d22c
--- /dev/null
+++ b/deno/tools/service.ts
@@ -0,0 +1,180 @@
+import { dirname, join, relative } from "@std/path";
+import { copySync, existsSync, walkSync } from "@std/fs";
+import { distinct } from "@std/collections";
+// @ts-types="npm:@types/mustache"
+import Mustache from "mustache";
+
+import { defineYargsModule, DEMAND_COMMAND_MESSAGE } from "./yargs.ts";
+
+const MUSTACHE_RENDER_OPTIONS: Mustache.RenderOptions = {
+ tags: ["@@", "@@"],
+ escape: (value: unknown) => String(value),
+};
+
+function mustacheParse(template: string) {
+ return Mustache.parse(template, MUSTACHE_RENDER_OPTIONS.tags);
+}
+
+function mustacheRender(template: string, view: Record<string, string>) {
+ return Mustache.render(template, view, {}, MUSTACHE_RENDER_OPTIONS);
+}
+
+function getVariableKeysOfTemplate(template: string): string[] {
+ return distinct(
+ mustacheParse(template)
+ .filter((v) => v[0] === "name")
+ .map((v) => v[1]),
+ );
+}
+
+function loadTemplatedConfigFiles(
+ files: string[],
+): Record<string, string> {
+ console.log("Scan config files ...");
+ const config: Record<string, string> = {};
+ for (const file of files) {
+ console.log(` from file ${file}`);
+ const text = Deno.readTextFileSync(file);
+ let lineNumber = 0;
+ for (const rawLine of text.split("\n")) {
+ lineNumber++;
+ const line = rawLine.trim();
+ if (line.length === 0) continue;
+ if (line.startsWith("#")) continue;
+ const equalSymbolIndex = line.indexOf("=");
+ if (equalSymbolIndex === -1) {
+ throw new Error(`Line ${lineNumber} of ${file} is invalid.`);
+ }
+ const [key, valueText] = [
+ line.slice(0, equalSymbolIndex).trim(),
+ line.slice(equalSymbolIndex + 1).trim(),
+ ];
+ console.log(` (${key in config ? "override" : "new"}) ${key}`);
+ getVariableKeysOfTemplate(valueText).forEach((name) => {
+ if (!(name in config)) {
+ throw new Error(
+ `Variable ${name} is not defined yet, perhaps due to typos or wrong order.`,
+ );
+ }
+ });
+ config[key] = mustacheRender(valueText, config);
+ }
+ }
+ return config;
+}
+
+const TEMPLATE_FILE_EXT = ".template";
+
+class TemplateDir {
+ templates: { path: string; ext: string; text: string; vars: string[] }[] = [];
+ plains: { path: string }[] = [];
+
+ constructor(public dir: string) {
+ console.log(`Scan template dir ${dir} ...`);
+ Array.from(
+ walkSync(dir, { includeDirs: false, followSymlinks: true }),
+ ).forEach(({ path }) => {
+ path = relative(this.dir, path);
+ if (path.endsWith(TEMPLATE_FILE_EXT)) {
+ console.log(` (template) ${path}`);
+ const text = Deno.readTextFileSync(join(dir, path));
+ this.templates.push({
+ path,
+ ext: TEMPLATE_FILE_EXT,
+ text,
+ vars: getVariableKeysOfTemplate(text),
+ });
+ } else {
+ console.log(` (plain) ${path}`);
+ this.plains.push({ path });
+ }
+ });
+ }
+
+ allNeededVars() {
+ return distinct(this.templates.flatMap((t) => t.vars));
+ }
+
+ generate(vars: Record<string, string>, generatedDir?: string) {
+ console.log(
+ `Generate to dir ${generatedDir ?? "[dry-run]"} ...`,
+ );
+
+ const undefinedVars = this.allNeededVars().filter((v) => !(v in vars));
+ if (undefinedVars.length !== 0) {
+ throw new Error(
+ `Needed variables are not defined: ${undefinedVars.join(", ")}`,
+ );
+ }
+
+ if (generatedDir != null) {
+ if (existsSync(generatedDir)) {
+ console.log(` delete old generated dir`);
+ Deno.removeSync(generatedDir, { recursive: true });
+ }
+
+ for (const file of this.plains) {
+ const [source, destination] = [
+ join(this.dir, file.path),
+ join(generatedDir, file.path),
+ ];
+ console.log(` copy ${file.path}`);
+ Deno.mkdirSync(dirname(destination), { recursive: true });
+ copySync(source, destination);
+ }
+ for (const file of this.templates) {
+ const path = file.path.slice(0, -file.ext.length);
+ const destination = join(generatedDir, path);
+ console.log(` generate ${path}`);
+ const rendered = mustacheRender(file.text, vars);
+ Deno.mkdirSync(dirname(destination), { recursive: true });
+ Deno.writeTextFileSync(destination, rendered);
+ }
+ }
+ }
+}
+
+export default defineYargsModule({
+ command: "service",
+ aliases: ["sv"],
+ describe: "Manage services.",
+ builder: (builder) => {
+ return builder
+ .option("project-dir", {
+ type: "string",
+ })
+ .demandOption("project-dir")
+ .command({
+ command: "gen-tmpl",
+ describe: "Generate files from templates",
+ builder: (builder) => {
+ return builder
+ .option("dry-run", {
+ type: "boolean",
+ default: true,
+ })
+ .strict();
+ },
+ handler: (argv) => {
+ const { projectDir, dryRun } = argv;
+
+ const config = loadTemplatedConfigFiles(
+ [
+ join(projectDir, "data/config"),
+ join(projectDir, "services/config.template"),
+ ],
+ );
+
+ new TemplateDir(
+ join(projectDir, "services/templates"),
+ ).generate(
+ config,
+ dryRun ? undefined : join(projectDir, "services/generated"),
+ );
+ console.log("Done!");
+ },
+ })
+ .demandCommand(1, DEMAND_COMMAND_MESSAGE);
+ },
+ handler: () => {},
+});
diff --git a/deno/tools/template.ts b/deno/tools/template.ts
deleted file mode 100644
index 0b043a1..0000000
--- a/deno/tools/template.ts
+++ /dev/null
@@ -1,122 +0,0 @@
-import { dirname, join, relative } from "@std/path";
-import { copySync, existsSync, walkSync } from "@std/fs";
-import { parse } from "@std/dotenv";
-import { distinct } from "@std/collections";
-// @ts-types="npm:@types/mustache"
-import Mustache from "mustache";
-
-Mustache.tags = ["@@", "@@"];
-Mustache.escape = (value) => String(value);
-
-function getVariableKeys(original: string): string[] {
- return distinct(
- Mustache.parse(original)
- .filter(function (v) {
- return v[0] === "name";
- })
- .map(function (v) {
- return v[1];
- }),
- );
-}
-
-export function loadVariables(files: string[]): Record<string, string> {
- const vars: Record<string, string> = {};
- for (const file of files) {
- const text = Deno.readTextFileSync(file);
- for (const [key, valueText] of Object.entries(parse(text))) {
- getVariableKeys(valueText).forEach((name) => {
- if (!(name in vars)) {
- throw new Error(
- `Variable ${name} is not defined yet, perhaps due to typos or wrong order.`,
- );
- }
- });
- vars[key] = Mustache.render(valueText, vars);
- }
- }
- return vars;
-}
-
-const TEMPLATE_FILE_EXT = ".template";
-
-export class TemplateDir {
- templates: { path: string; ext: string; text: string; vars: string[] }[] = [];
- plains: { path: string }[] = [];
-
- constructor(public dir: string) {
- console.log("Scanning template dir:");
- Array.from(
- walkSync(dir, { includeDirs: false, followSymlinks: true }),
- ).forEach(({ path }) => {
- path = relative(this.dir, path);
- if (path.endsWith(TEMPLATE_FILE_EXT)) {
- console.log(` (template) ${path}`);
- const text = Deno.readTextFileSync(join(dir, path));
- this.templates.push({
- path,
- ext: TEMPLATE_FILE_EXT,
- text,
- vars: getVariableKeys(text),
- });
- } else {
- console.log(` (plain) ${path}`);
- this.plains.push({ path });
- }
- });
- console.log("Done scanning template dir.");
- }
-
- allNeededVars() {
- return distinct(this.templates.flatMap((t) => t.vars));
- }
-
- generate(vars: Record<string, string>, generatedDir?: string) {
- console.log(
- `Generating, template dir: ${this.dir}, generated dir: ${generatedDir ?? "[dry-run]"}:`,
- );
-
- const undefinedVars = this.allNeededVars().filter((v) => !(v in vars));
- if (undefinedVars.length !== 0) {
- throw new Error(
- `Needed variables are not defined: ${undefinedVars.join(", ")}`,
- );
- }
-
- if (generatedDir != null) {
- if (existsSync(generatedDir)) {
- console.log(` delete old generated dir ${generatedDir}`);
- Deno.removeSync(generatedDir, { recursive: true });
- }
-
- for (const file of this.plains) {
- const [source, destination] = [
- join(this.dir, file.path),
- join(generatedDir, file.path),
- ];
- console.log(` copy ${source} to ${destination} ...`);
- Deno.mkdirSync(dirname(destination), { recursive: true });
- copySync(source, destination);
- }
- for (const file of this.templates) {
- const [source, destination] = [
- join(this.dir, file.path),
- join(generatedDir, file.path.slice(0, -file.ext.length)),
- ];
- console.log(` generate ${source} to ${destination} ...`);
- const rendered = Mustache.render(file.text, vars);
- Deno.mkdirSync(dirname(destination), { recursive: true });
- Deno.writeTextFileSync(destination, rendered);
- }
- }
- console.log(`Done generating.`);
- }
-
- generateWithVariableFiles(varFiles: string[], generatedDir?: string) {
- console.log("Scanning defined vars:");
- const vars = loadVariables(varFiles);
- Object.keys(vars).forEach((name) => console.log(` ${name}`));
- console.log("Done scanning defined vars.");
- this.generate(vars, generatedDir);
- }
-}
diff --git a/deno/tools/vm.ts b/deno/tools/vm.ts
new file mode 100644
index 0000000..b54c0d4
--- /dev/null
+++ b/deno/tools/vm.ts
@@ -0,0 +1,225 @@
+import os from "node:os";
+import { join } from "@std/path";
+import { defineYargsModule, DEMAND_COMMAND_MESSAGE } from "./yargs.ts";
+
+type ArchAliasMap = { [name: string]: string[] };
+const arches = {
+ x86_64: ["x86_64", "amd64"],
+ i386: ["i386", "x86", "i686"],
+} as const satisfies ArchAliasMap;
+type Arch = keyof typeof arches;
+type GeneralArch = (typeof arches)[Arch][number];
+
+function normalizeArch(generalName: GeneralArch): Arch {
+ for (const [name, aliases] of Object.entries(arches as ArchAliasMap)) {
+ if (aliases.includes(generalName)) return name as Arch;
+ }
+ throw Error("Unknown architecture name.");
+}
+
+interface GeneralVmSetup {
+ name?: string[];
+ arch: GeneralArch;
+ cpuNumber?: number;
+ memory?: number;
+ disk: string;
+ usbTablet?: boolean;
+ sshForwardPort?: number;
+ tpm?: boolean;
+ kvm?: boolean;
+}
+
+interface VmSetup {
+ arch: Arch;
+ cpuNumber: number;
+ memory: number;
+ disk: string;
+ usbTablet: boolean;
+ sshForwardPort?: number;
+ tpm: boolean;
+ kvm: boolean;
+}
+
+const VM_DIR = join(os.homedir(), "vms");
+
+function getDiskFilePath(name: string): string {
+ return join(VM_DIR, `${name}.qcow2`);
+}
+
+const MY_VMS: GeneralVmSetup[] = [
+ {
+ name: ["hurd", ...arches.i386.map((a) => `hurd-${a}`)],
+ arch: "i386",
+ disk: getDiskFilePath("hurd-i386"),
+ sshForwardPort: 3222,
+ },
+ {
+ name: [...arches.x86_64.map((a) => `hurd-${a}`)],
+ arch: "x86_64",
+ disk: getDiskFilePath("hurd-x86_64"),
+ sshForwardPort: 3223,
+ },
+ {
+ name: ["win"],
+ arch: "x86_64",
+ cpuNumber: 4,
+ memory: 16,
+ disk: getDiskFilePath("win"),
+ usbTablet: true,
+ tpm: true,
+ },
+];
+
+function normalizeVmSetup(generalSetup: GeneralVmSetup): VmSetup {
+ const { arch, cpuNumber, memory, disk, usbTablet, sshForwardPort, tpm, kvm } =
+ generalSetup;
+
+ const normalizedArch = normalizeArch(arch);
+ const is64 = normalizedArch === "x86_64";
+
+ return {
+ arch: normalizedArch,
+ disk,
+ cpuNumber: cpuNumber ?? 1,
+ memory: memory ?? (is64 ? 8 : 4),
+ usbTablet: usbTablet ?? false,
+ sshForwardPort,
+ tpm: tpm ?? false,
+ kvm: kvm ?? Deno.build.os === "linux",
+ };
+}
+
+function resolveVmSetup(
+ name: string,
+ vms: GeneralVmSetup[],
+): VmSetup | undefined {
+ const setup = vms.find((vm) => vm.name?.includes(name));
+ return setup == null ? undefined : normalizeVmSetup(setup);
+}
+
+const qemuBinPrefix = "qemu-system" as const;
+
+const qemuBinSuffix = {
+ x86_64: "x86_64",
+ i386: "x86_64",
+} as const;
+
+function getQemuBin(arch: Arch): string {
+ return `${qemuBinPrefix}-${qemuBinSuffix[arch]}`;
+}
+
+function getLinuxHostArgs(kvm: boolean): string[] {
+ return kvm ? ["-enable-kvm"] : [];
+}
+
+function getMachineArgs(vm: VmSetup): string[] {
+ const is64 = vm.arch === "x86_64";
+ const machineArgs = is64 ? ["-machine", "q35"] : [];
+ return [...machineArgs, "-smp", String(vm.cpuNumber), "-m", `${vm.memory}G`];
+}
+
+function getDeviceArgs(vm: VmSetup): string[] {
+ const { usbTablet } = vm;
+ return usbTablet ? ["-usb", "-device", "usb-tablet"] : [];
+}
+
+function getNetworkArgs(sshForwardPort?: number): string[] {
+ const args = ["-net", "nic"];
+ if (sshForwardPort != null) {
+ args.push("-net", `user,hostfwd=tcp::${sshForwardPort}-:22`);
+ }
+ return args;
+}
+
+function getDisplayArgs(): string[] {
+ return ["-vga", "vmware"];
+}
+
+function getDiskArgs(disk: string): string[] {
+ return ["-drive", `cache=writeback,file=${disk}`];
+}
+
+function getTpmControlSocketPath(): string {
+ return join(VM_DIR, "tpm2/swtpm-sock");
+}
+
+function getTpmArgs(tpm: boolean): string[] {
+ if (!tpm) return [];
+ return [
+ "-chardev",
+ `socket,id=chrtpm,path=${getTpmControlSocketPath()}`,
+ "-tpmdev",
+ "emulator,id=tpm0,chardev=chrtpm",
+ "-device",
+ "tpm-tis,tpmdev=tpm0",
+ ];
+}
+
+function getTpmPreCommand(): string[] {
+ return [
+ "swtpm",
+ "socket",
+ "--tpm2",
+ "--tpmstate",
+ `dir=${join(VM_DIR, "tpm2")}`,
+ "--ctrl",
+ `type=unixio,path=${getTpmControlSocketPath()}`,
+ ];
+}
+
+function createPreCommands(setup: VmSetup): string[][] {
+ const { tpm } = setup;
+ const result = [];
+ if (tpm) result.push(getTpmPreCommand());
+ return result;
+}
+
+function createQemuArgs(setup: VmSetup): string[] {
+ const { arch, disk, sshForwardPort, tpm } = setup;
+ return [
+ getQemuBin(arch),
+ ...getLinuxHostArgs(setup.kvm),
+ ...getMachineArgs(setup),
+ ...getDeviceArgs(setup),
+ ...getDisplayArgs(),
+ ...getNetworkArgs(sshForwardPort),
+ ...getDiskArgs(disk),
+ ...getTpmArgs(tpm),
+ ];
+}
+
+const gen = defineYargsModule({
+ command: "gen <name>",
+ describe: "generate cli command to run the vm",
+ builder: (builder) => {
+ return builder
+ .positional("name", {
+ describe: "name of the vm to run",
+ type: "string",
+ })
+ .demandOption("name")
+ .strict();
+ },
+ handler: (argv) => {
+ const vm = resolveVmSetup(argv.name, MY_VMS);
+ if (vm == null) {
+ console.error(`No vm called ${argv.name} is found.`);
+ Deno.exit(-1);
+ }
+ const preCommands = createPreCommands(vm);
+ const cli = createQemuArgs(vm);
+ for (const command of preCommands) {
+ console.log(`${command.join(" ")} &`);
+ }
+ console.log(`${cli.join(" ")}`);
+ },
+});
+
+export default defineYargsModule({
+ command: "vm",
+ describe: "Manage (qemu) virtual machines.",
+ builder: (builder) => {
+ return builder.command(gen).demandCommand(1, DEMAND_COMMAND_MESSAGE);
+ },
+ handler: () => {},
+});
diff --git a/deno/tools/yargs.ts b/deno/tools/yargs.ts
new file mode 100644
index 0000000..eaa7803
--- /dev/null
+++ b/deno/tools/yargs.ts
@@ -0,0 +1,12 @@
+// @ts-types="npm:@types/yargs"
+export { default } from "yargs";
+export * from "yargs";
+
+import { CommandModule } from "yargs";
+export function defineYargsModule<T, U>(
+ module: CommandModule<T, U>,
+): CommandModule<T, U> {
+ return module;
+}
+
+export const DEMAND_COMMAND_MESSAGE = "No command is specified";
generated by cgit v1.2.3 (git 2.46.0) at 2025-12-14 20:01:32 +0000