aboutsummaryrefslogtreecommitdiff
path: root/deno
diff options
context:
space:
mode:
Diffstat (limited to 'deno')
-rw-r--r--deno/base/config.ts6
-rw-r--r--deno/base/date.ts6
-rw-r--r--deno/base/deno.json6
-rw-r--r--deno/base/lib.ts30
-rw-r--r--deno/base/log.ts112
-rw-r--r--deno/base/text.ts3
-rw-r--r--deno/deno.json11
-rw-r--r--deno/deno.lock97
-rw-r--r--deno/mail-relay/aws/deliver.ts63
-rw-r--r--deno/mail-relay/aws/mail.ts53
-rw-r--r--deno/mail-relay/dovecot.ts95
-rw-r--r--deno/mail-relay/mail.ts349
-rw-r--r--deno/mail/app.ts (renamed from deno/mail-relay/app.ts)36
-rw-r--r--deno/mail/aws/app.ts (renamed from deno/mail-relay/aws/app.ts)223
-rw-r--r--deno/mail/aws/deliver.ts57
-rw-r--r--deno/mail/aws/fetch.ts (renamed from deno/mail-relay/aws/fetch.ts)91
-rw-r--r--deno/mail/db.test.ts (renamed from deno/mail-relay/db.test.ts)8
-rw-r--r--deno/mail/db.ts (renamed from deno/mail-relay/db.ts)38
-rw-r--r--deno/mail/deno.json (renamed from deno/mail-relay/deno.json)2
-rw-r--r--deno/mail/dovecot.ts219
-rw-r--r--deno/mail/dumb-smtp-server.ts (renamed from deno/mail-relay/dumb-smtp-server.ts)78
-rw-r--r--deno/mail/mail-parsing.ts144
-rw-r--r--deno/mail/mail.test.ts (renamed from deno/mail-relay/mail.test.ts)33
-rw-r--r--deno/mail/mail.ts304
-rw-r--r--deno/service-manager/deno.json12
-rw-r--r--deno/service-manager/main.ts39
-rw-r--r--deno/service-manager/template.ts122
-rw-r--r--deno/tools/deno.json1
-rw-r--r--deno/tools/geosite.ts (renamed from deno/tools/gen-geosite-rules.ts)76
-rw-r--r--deno/tools/main.ts14
-rw-r--r--deno/tools/service.ts180
-rw-r--r--deno/tools/vm.ts225
-rw-r--r--deno/tools/yargs.ts12
33 files changed, 1569 insertions, 1176 deletions
diff --git a/deno/base/config.ts b/deno/base/config.ts
index 8fce1d8..96cc869 100644
--- a/deno/base/config.ts
+++ b/deno/base/config.ts
@@ -1,4 +1,4 @@
-import { camelCaseToKebabCase } from "./text.ts";
+import { StringUtils } from "./lib.ts";
export interface ConfigDefinitionItem {
readonly description: string;
@@ -29,7 +29,9 @@ export class ConfigProvider<K extends string> {
for (const [key, def] of Object.entries(definition as ConfigDefinition)) {
map[key] = {
...def,
- env: `${this.#prefix}-${camelCaseToKebabCase(key as string)}`
+ env: `${this.#prefix}-${
+ StringUtils.camelCaseToKebabCase(key as string)
+ }`
.replaceAll("-", "_")
.toUpperCase(),
};
diff --git a/deno/base/date.ts b/deno/base/date.ts
deleted file mode 100644
index e65691e..0000000
--- a/deno/base/date.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-export function toFileNameString(date: Date, dateOnly?: boolean): string {
- const str = date.toISOString();
- return dateOnly === true
- ? str.slice(0, str.indexOf("T"))
- : str.replaceAll(/:|\./g, "-");
-}
diff --git a/deno/base/deno.json b/deno/base/deno.json
index 2c2d550..582f0f6 100644
--- a/deno/base/deno.json
+++ b/deno/base/deno.json
@@ -2,10 +2,8 @@
"name": "@crupest/base",
"version": "0.1.0",
"exports": {
+ ".": "./lib.ts",
"./config": "./config.ts",
- "./cron": "./cron.ts",
- "./date": "./date.ts",
- "./text": "./text.ts",
- "./log": "./log.ts"
+ "./cron": "./cron.ts"
}
}
diff --git a/deno/base/lib.ts b/deno/base/lib.ts
new file mode 100644
index 0000000..af75115
--- /dev/null
+++ b/deno/base/lib.ts
@@ -0,0 +1,30 @@
+function camelCaseToKebabCase(str: string): string {
+ return str.replace(/[A-Z]/g, (m) => "-" + m.toLowerCase());
+}
+
+function prependNonEmpty<T>(
+ object: T | null | undefined,
+ prefix: string = " ",
+): string {
+ if (object == null) return "";
+ const string = typeof object === "string" ? object : String(object);
+ return string.length === 0 ? "" : prefix + string;
+}
+
+export const StringUtils = Object.freeze({
+ camelCaseToKebabCase,
+ prependNonEmpty,
+});
+
+function toFileNameString(date: Date, dateOnly?: boolean): string {
+ const str = date.toISOString();
+ return dateOnly === true
+ ? str.slice(0, str.indexOf("T"))
+ : str.replaceAll(/:|\./g, "-");
+}
+
+export const DateUtils = Object.freeze(
+ {
+ toFileNameString,
+ } as const,
+);
diff --git a/deno/base/log.ts b/deno/base/log.ts
deleted file mode 100644
index cc71dfa..0000000
--- a/deno/base/log.ts
+++ /dev/null
@@ -1,112 +0,0 @@
-import { join } from "@std/path";
-
-import { toFileNameString } from "./date.ts";
-
-export type LogLevel = "error" | "warn" | "info";
-
-export interface LogOptions {
- level?: LogLevel;
- cause?: unknown;
-}
-
-export interface ExternalLogStream extends Disposable {
- stream: WritableStream;
-}
-
-export class Logger {
- #defaultLevel = "info" as const;
- #externalLogDir?: string;
-
- get externalLogDir() {
- return this.#externalLogDir;
- }
-
- set externalLogDir(value: string | undefined) {
- this.#externalLogDir = value;
- if (value != null) {
- Deno.mkdirSync(value, {
- recursive: true,
- });
- }
- }
-
- write(message: string, options?: LogOptions): void {
- const logFunction = console[options?.level ?? this.#defaultLevel];
- if (options?.cause != null) {
- logFunction.call(console, message, options.cause);
- } else {
- logFunction.call(console, message);
- }
- }
-
- info(message: string) {
- this.write(message, { level: "info" });
- }
-
- tagInfo(tag: string, message: string) {
- this.info(tag + " " + message);
- }
-
- warn(message: string) {
- this.write(message, { level: "warn" });
- }
-
- tagWarn(tag: string, message: string) {
- this.warn(tag + " " + message);
- }
-
- error(message: string, cause?: unknown) {
- this.write(message, { level: "info", cause });
- }
-
- tagError(tag: string, message: string, cause?: unknown) {
- this.error(tag + " " + message, cause);
- }
-
- async createExternalLogStream(
- name: string,
- options?: {
- noTime?: boolean;
- },
- ): Promise<ExternalLogStream> {
- if (name.includes("/")) {
- throw new Error(`External log stream's name (${name}) contains '/'.`);
- }
- if (this.#externalLogDir == null) {
- throw new Error("External log directory is not set.");
- }
-
- const logPath = join(
- this.#externalLogDir,
- options?.noTime === true
- ? name
- : `${name}-${toFileNameString(new Date())}`,
- );
-
- const file = await Deno.open(logPath, {
- read: false,
- write: true,
- append: true,
- create: true,
- });
- return {
- stream: file.writable,
- [Symbol.dispose]: file[Symbol.dispose].bind(file),
- };
- }
-
- async createExternalLogStreamsForProgram(
- program: string,
- ): Promise<{ stdout: WritableStream; stderr: WritableStream } & Disposable> {
- const stdout = await this.createExternalLogStream(`${program}-stdout`);
- const stderr = await this.createExternalLogStream(`${program}-stderr`);
- return {
- stdout: stdout.stream,
- stderr: stderr.stream,
- [Symbol.dispose]: () => {
- stdout[Symbol.dispose]();
- stderr[Symbol.dispose]();
- },
- };
- }
-}
diff --git a/deno/base/text.ts b/deno/base/text.ts
deleted file mode 100644
index f3e4020..0000000
--- a/deno/base/text.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-export function camelCaseToKebabCase(str: string): string {
- return str.replace(/[A-Z]/g, (m) => "-" + m.toLowerCase());
-}
diff --git a/deno/deno.json b/deno/deno.json
index 55ffcb8..286451e 100644
--- a/deno/deno.json
+++ b/deno/deno.json
@@ -1,17 +1,18 @@
{
- "workspace": ["./base", "./service-manager", "./mail-relay", "./tools" ],
+ "workspace": ["./base", "./mail", "./tools"],
"tasks": {
- "compile:mail-relay": "deno task --cwd=mail-relay compile",
- "compile:service-manager": "deno task --cwd=service-manager compile"
+ "compile:mail": "deno task --cwd=mail compile"
},
"imports": {
- "@std/cli": "jsr:@std/cli@^1.0.19",
"@std/collections": "jsr:@std/collections@^1.1.1",
"@std/csv": "jsr:@std/csv@^1.0.6",
"@std/encoding": "jsr:@std/encoding@^1.0.10",
"@std/expect": "jsr:@std/expect@^1.0.16",
"@std/io": "jsr:@std/io@^0.225.2",
"@std/path": "jsr:@std/path@^1.1.0",
- "@std/testing": "jsr:@std/testing@^1.0.13"
+ "@std/testing": "jsr:@std/testing@^1.0.13",
+ "@std/fs": "jsr:@std/fs@^1.0.18",
+ "yargs": "npm:yargs@^18.0.0",
+ "@types/yargs": "npm:@types/yargs@^17.0.33"
}
}
diff --git a/deno/deno.lock b/deno/deno.lock
index 357b31f..bdc8c3f 100644
--- a/deno/deno.lock
+++ b/deno/deno.lock
@@ -7,11 +7,9 @@
"jsr:@std/assert@^1.0.13": "1.0.13",
"jsr:@std/async@^1.0.13": "1.0.13",
"jsr:@std/bytes@^1.0.5": "1.0.6",
- "jsr:@std/cli@^1.0.19": "1.0.19",
"jsr:@std/collections@^1.1.1": "1.1.1",
"jsr:@std/csv@^1.0.6": "1.0.6",
"jsr:@std/data-structures@^1.0.8": "1.0.8",
- "jsr:@std/dotenv@~0.225.5": "0.225.5",
"jsr:@std/encoding@1": "1.0.10",
"jsr:@std/encoding@^1.0.10": "1.0.10",
"jsr:@std/expect@^1.0.16": "1.0.16",
@@ -36,10 +34,13 @@
"npm:@types/lodash@*": "4.17.17",
"npm:@types/mustache@*": "4.2.6",
"npm:@types/node@*": "22.15.15",
+ "npm:@types/yargs@*": "17.0.33",
+ "npm:@types/yargs@^17.0.33": "17.0.33",
"npm:email-addresses@5": "5.0.0",
"npm:hono@^4.7.11": "4.7.11",
"npm:kysely@~0.28.2": "0.28.2",
"npm:mustache@^4.2.0": "4.2.0",
+ "npm:yargs@18": "18.0.0",
"npm:zod@^3.25.48": "3.25.51"
},
"jsr": {
@@ -74,9 +75,6 @@
"@std/bytes@1.0.6": {
"integrity": "f6ac6adbd8ccd99314045f5703e23af0a68d7f7e58364b47d2c7f408aeb5820a"
},
- "@std/cli@1.0.19": {
- "integrity": "b3601a54891f89f3f738023af11960c4e6f7a45dc76cde39a6861124cba79e88"
- },
"@std/collections@1.1.1": {
"integrity": "eff6443fbd9d5a6697018fb39c5d13d5f662f0045f21392d640693d0008ab2af"
},
@@ -89,9 +87,6 @@
"@std/data-structures@1.0.8": {
"integrity": "2fb7219247e044c8fcd51341788547575653c82ae2c759ff209e0263ba7d9b66"
},
- "@std/dotenv@0.225.5": {
- "integrity": "9ce6f9d0ec3311f74a32535aa1b8c62ed88b1ab91b7f0815797d77a6f60c922f"
- },
"@std/encoding@1.0.10": {
"integrity": "8783c6384a2d13abd5e9e87a7ae0520a30e9f56aeeaa3bdf910a3eaaf5c811a1"
},
@@ -1183,12 +1178,41 @@
"undici-types"
]
},
+ "@types/yargs-parser@21.0.3": {
+ "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ=="
+ },
+ "@types/yargs@17.0.33": {
+ "integrity": "sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==",
+ "dependencies": [
+ "@types/yargs-parser"
+ ]
+ },
+ "ansi-regex@6.1.0": {
+ "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA=="
+ },
+ "ansi-styles@6.2.1": {
+ "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug=="
+ },
"bowser@2.11.0": {
"integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA=="
},
+ "cliui@9.0.1": {
+ "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==",
+ "dependencies": [
+ "string-width",
+ "strip-ansi",
+ "wrap-ansi"
+ ]
+ },
"email-addresses@5.0.0": {
"integrity": "sha512-4OIPYlA6JXqtVn8zpHpGiI7vE6EQOAg16aGnDMIAlZVinnoZ8208tW1hAbjWydgN/4PLTT9q+O1K6AH/vALJGw=="
},
+ "emoji-regex@10.4.0": {
+ "integrity": "sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw=="
+ },
+ "escalade@3.2.0": {
+ "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="
+ },
"fast-xml-parser@4.4.1": {
"integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==",
"dependencies": [
@@ -1196,6 +1220,12 @@
],
"bin": true
},
+ "get-caller-file@2.0.5": {
+ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
+ },
+ "get-east-asian-width@1.3.0": {
+ "integrity": "sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ=="
+ },
"hono@4.7.11": {
"integrity": "sha512-rv0JMwC0KALbbmwJDEnxvQCeJh+xbS3KEWW5PC9cMJ08Ur9xgatI0HmtgYZfOdOSOeYsp5LO2cOhdI8cLEbDEQ=="
},
@@ -1206,6 +1236,20 @@
"integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==",
"bin": true
},
+ "string-width@7.2.0": {
+ "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==",
+ "dependencies": [
+ "emoji-regex",
+ "get-east-asian-width",
+ "strip-ansi"
+ ]
+ },
+ "strip-ansi@7.1.0": {
+ "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
+ "dependencies": [
+ "ansi-regex"
+ ]
+ },
"strnum@1.1.2": {
"integrity": "sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA=="
},
@@ -1219,23 +1263,50 @@
"integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
"bin": true
},
+ "wrap-ansi@9.0.0": {
+ "integrity": "sha512-G8ura3S+3Z2G+mkgNRq8dqaFZAuxfsxpBB8OCTGRTCtp+l/v9nbFNmCUP1BZMts3G1142MsZfn6eeUKrr4PD1Q==",
+ "dependencies": [
+ "ansi-styles",
+ "string-width",
+ "strip-ansi"
+ ]
+ },
+ "y18n@5.0.8": {
+ "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="
+ },
+ "yargs-parser@22.0.0": {
+ "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw=="
+ },
+ "yargs@18.0.0": {
+ "integrity": "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==",
+ "dependencies": [
+ "cliui",
+ "escalade",
+ "get-caller-file",
+ "string-width",
+ "y18n",
+ "yargs-parser"
+ ]
+ },
"zod@3.25.51": {
"integrity": "sha512-TQSnBldh+XSGL+opiSIq0575wvDPqu09AqWe1F7JhUMKY+M91/aGlK4MhpVNO7MgYfHcVCB1ffwAUTJzllKJqg=="
}
},
"workspace": {
"dependencies": [
- "jsr:@std/cli@^1.0.19",
"jsr:@std/collections@^1.1.1",
"jsr:@std/csv@^1.0.6",
"jsr:@std/encoding@^1.0.10",
"jsr:@std/expect@^1.0.16",
+ "jsr:@std/fs@^1.0.18",
"jsr:@std/io@~0.225.2",
"jsr:@std/path@^1.1.0",
- "jsr:@std/testing@^1.0.13"
+ "jsr:@std/testing@^1.0.13",
+ "npm:@types/yargs@^17.0.33",
+ "npm:yargs@18"
],
"members": {
- "mail-relay": {
+ "mail": {
"dependencies": [
"jsr:@db/sqlite@0.12",
"npm:@aws-sdk/client-s3@^3.821.0",
@@ -1248,10 +1319,8 @@
"npm:zod@^3.25.48"
]
},
- "service-manager": {
+ "tools": {
"dependencies": [
- "jsr:@std/dotenv@~0.225.5",
- "jsr:@std/fs@^1.0.18",
"npm:mustache@^4.2.0"
]
}
diff --git a/deno/mail-relay/aws/deliver.ts b/deno/mail-relay/aws/deliver.ts
deleted file mode 100644
index 9950e37..0000000
--- a/deno/mail-relay/aws/deliver.ts
+++ /dev/null
@@ -1,63 +0,0 @@
-// spellchecker:words sesv2 amazonses
-
-import {
- SendEmailCommand,
- SESv2Client,
- SESv2ClientConfig,
-} from "@aws-sdk/client-sesv2";
-
-import { Logger } from "@crupest/base/log";
-import { Mail, MailDeliverContext, SyncMailDeliverer } from "../mail.ts";
-
-declare module "../mail.ts" {
- interface MailDeliverResult {
- awsMessageId?: string;
- }
-}
-
-export class AwsMailDeliverer extends SyncMailDeliverer {
- readonly name = "aws";
- readonly #logger;
- readonly #aws;
- readonly #ses;
-
- constructor(logger: Logger, aws: SESv2ClientConfig) {
- super(logger);
- this.#logger = logger;
- this.#aws = aws;
- this.#ses = new SESv2Client(aws);
- }
-
- protected override async doDeliver(
- mail: Mail,
- context: MailDeliverContext,
- ): Promise<void> {
- this.#logger.info("Begin to call aws send-email api...");
-
- try {
- const sendCommand = new SendEmailCommand({
- Content: {
- Raw: { Data: mail.toUtf8Bytes() },
- },
- });
-
- const res = await this.#ses.send(sendCommand);
- if (res.MessageId == null) {
- this.#logger.warn("Aws send-email returns no message id.");
- } else {
- context.result.awsMessageId = `${res.MessageId}@${this.#aws.region}.amazonses.com`;
- }
-
- context.result.recipients.set("*", {
- kind: "done",
- message: `Successfully called aws send-email, message id ${context.result.awsMessageId}.`,
- });
- } catch (cause) {
- context.result.recipients.set("*", {
- kind: "fail",
- message: "An error was thrown when calling aws send-email." + cause,
- cause,
- });
- }
- }
-}
diff --git a/deno/mail-relay/aws/mail.ts b/deno/mail-relay/aws/mail.ts
deleted file mode 100644
index d2cfad1..0000000
--- a/deno/mail-relay/aws/mail.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-import { MailDeliverContext, MailDeliverHook } from "../mail.ts";
-
-export class AwsMailMessageIdRewriteHook implements MailDeliverHook {
- readonly #lookup;
-
- constructor(lookup: (origin: string) => Promise<string | null>) {
- this.#lookup = lookup;
- }
-
- async callback(context: MailDeliverContext): Promise<void> {
- context.logger.info("Rewrite message ids...");
- const addresses = context.mail.simpleFindAllAddresses();
- context.logger.info(`Addresses found in mail: ${addresses.join(", ")}.`);
- for (const address of addresses) {
- const awsMessageId = await this.#lookup(address);
- if (awsMessageId != null && awsMessageId.length !== 0) {
- context.logger.info(`Rewrite ${address} to ${awsMessageId}.`);
- context.mail.raw = context.mail.raw.replaceAll(address, awsMessageId);
- }
- }
- context.logger.info("Done rewrite message ids.");
- }
-}
-
-export class AwsMailMessageIdSaveHook implements MailDeliverHook {
- readonly #record;
-
- constructor(record: (original: string, aws: string) => Promise<void>) {
- this.#record = record;
- }
-
- async callback(context: MailDeliverContext): Promise<void> {
- context.logger.info("Save aws message ids...");
- const messageId = context.mail
- .startSimpleParse(context.logger)
- .sections()
- .headers()
- .messageId();
- if (messageId == null) {
- context.logger.info(
- "Original mail does not have message id. Skip saving.",
- );
- return;
- }
- if (context.result.awsMessageId != null) {
- context.logger.info(
- `Saving ${messageId} => ${context.result.awsMessageId}.`,
- );
- await this.#record(messageId, context.result.awsMessageId);
- }
- context.logger.info("Done save message ids.");
- }
-}
diff --git a/deno/mail-relay/dovecot.ts b/deno/mail-relay/dovecot.ts
deleted file mode 100644
index 124a82b..0000000
--- a/deno/mail-relay/dovecot.ts
+++ /dev/null
@@ -1,95 +0,0 @@
-import { basename } from "@std/path";
-
-import { Logger } from "@crupest/base/log";
-
-import { Mail, MailDeliverContext, MailDeliverer } from "./mail.ts";
-
-export class DovecotMailDeliverer extends MailDeliverer {
- readonly name = "dovecot";
- readonly #ldaPath;
-
- constructor(logger: Logger, ldaPath: string) {
- super(logger);
- this.#ldaPath = ldaPath;
- }
-
- protected override async doDeliver(
- mail: Mail,
- context: MailDeliverContext,
- ): Promise<void> {
- const ldaPath = this.#ldaPath;
- const ldaBinName = basename(ldaPath);
- const utf8Stream = mail.toUtf8Bytes();
-
- const recipients = [...context.recipients];
-
- if (recipients.length === 0) {
- context.result.message =
- "Failed to deliver to dovecot, no recipients are specified.";
- return;
- }
-
- this.logger.info(`Deliver to dovecot users: ${recipients.join(", ")}.`);
-
- for (const recipient of recipients) {
- try {
- const commandArgs = ["-d", recipient];
- this.logger.info(`Run ${ldaBinName} ${commandArgs.join(" ")}...`);
-
- const ldaCommand = new Deno.Command(ldaPath, {
- args: commandArgs,
- stdin: "piped",
- stdout: "piped",
- stderr: "piped",
- });
-
- const ldaProcess = ldaCommand.spawn();
- using logFiles =
- await this.logger.createExternalLogStreamsForProgram(ldaBinName);
- ldaProcess.stdout.pipeTo(logFiles.stdout);
- ldaProcess.stderr.pipeTo(logFiles.stderr);
-
- const stdinWriter = ldaProcess.stdin.getWriter();
- await stdinWriter.write(utf8Stream);
- await stdinWriter.close();
-
- const status = await ldaProcess.status;
-
- if (status.success) {
- context.result.recipients.set(recipient, {
- kind: "done",
- message: `${ldaBinName} exited with success.`,
- });
- } else {
- let message = `${ldaBinName} exited with error code ${status.code}`;
-
- if (status.signal != null) {
- message += ` (signal ${status.signal})`;
- }
-
- // https://doc.dovecot.org/main/core/man/dovecot-lda.1.html
- switch (status.code) {
- case 67:
- message += ", recipient user not known";
- break;
- case 75:
- message += ", temporary error";
- break;
- }
-
- message += ".";
-
- context.result.recipients.set(recipient, { kind: "fail", message });
- }
- } catch (cause) {
- context.result.recipients.set(recipient, {
- kind: "fail",
- message: "An error is thrown when running lda: " + cause,
- cause,
- });
- }
- }
-
- this.logger.info("Done handling all recipients.");
- }
-}
diff --git a/deno/mail-relay/mail.ts b/deno/mail-relay/mail.ts
deleted file mode 100644
index 12d5972..0000000
--- a/deno/mail-relay/mail.ts
+++ /dev/null
@@ -1,349 +0,0 @@
-import { encodeBase64 } from "@std/encoding/base64";
-import { parse } from "@std/csv/parse";
-import emailAddresses from "email-addresses";
-
-import { Logger } from "@crupest/base/log";
-
-class MailSimpleParseError extends Error {}
-
-class MailSimpleParsedHeaders {
- #logger;
-
- constructor(
- logger: Logger | undefined,
- public fields: [key: string, value: string][],
- ) {
- this.#logger = logger;
- }
-
- getFirst(fieldKey: string): string | undefined {
- for (const [key, value] of this.fields) {
- if (key.toLowerCase() === fieldKey.toLowerCase()) return value;
- }
- return undefined;
- }
-
- messageId(): string | undefined {
- const messageIdField = this.getFirst("message-id");
- if (messageIdField == null) return undefined;
-
- const match = messageIdField.match(/\<(.*?)\>/);
- if (match != null) {
- return match[1];
- } else {
- this.#logger?.warn(
- "Invalid message-id header of mail: " + messageIdField,
- );
- return undefined;
- }
- }
-
- date(invalidToUndefined: boolean = true): Date | undefined {
- const dateField = this.getFirst("date");
- if (dateField == null) return undefined;
-
- const date = new Date(dateField);
- if (invalidToUndefined && isNaN(date.getTime())) {
- this.#logger?.warn(`Invalid date string (${dateField}) found in header.`);
- return undefined;
- }
- return date;
- }
-
- recipients(options?: { domain?: string; headers?: string[] }): Set<string> {
- const domain = options?.domain;
- const headers = options?.headers ?? ["to", "cc", "bcc", "x-original-to"];
- const recipients = new Set<string>();
- for (const [key, value] of this.fields) {
- if (headers.includes(key.toLowerCase())) {
- emailAddresses
- .parseAddressList(value)
- ?.flatMap((a) => (a.type === "mailbox" ? a : a.addresses))
- ?.forEach(({ address }) => {
- if (domain == null || address.endsWith(domain)) {
- recipients.add(address);
- }
- });
- }
- }
- return recipients;
- }
-}
-
-class MailSimpleParsedSections {
- header: string;
- body: string;
- eol: string;
- sep: string;
-
- #logger;
-
- constructor(logger: Logger | undefined, raw: string) {
- this.#logger = logger;
-
- const twoEolMatch = raw.match(/(\r?\n)(\r?\n)/);
- if (twoEolMatch == null) {
- throw new MailSimpleParseError(
- "No header/body section separator (2 successive EOLs) found.",
- );
- }
-
- const [eol, sep] = [twoEolMatch[1], twoEolMatch[2]];
-
- if (eol !== sep) {
- logger?.warn("Different EOLs (\\r\\n, \\n) found.");
- }
-
- this.header = raw.slice(0, twoEolMatch.index!);
- this.body = raw.slice(twoEolMatch.index! + eol.length + sep.length);
- this.eol = eol;
- this.sep = sep;
- }
-
- headers(): MailSimpleParsedHeaders {
- const headers = [] as [key: string, value: string][];
-
- let field: string | null = null;
- let lineNumber = 1;
-
- const handleField = () => {
- if (field == null) return;
- const sepPos = field.indexOf(":");
- if (sepPos === -1) {
- throw new MailSimpleParseError(`No ':' in the header line: ${field}`);
- }
- headers.push([field.slice(0, sepPos).trim(), field.slice(sepPos + 1)]);
- field = null;
- };
-
- for (const line of this.header.trimEnd().split(/\r?\n|\r/)) {
- if (line.match(/^\s/)) {
- if (field == null) {
- throw new MailSimpleParseError("Header section starts with a space.");
- }
- field += line;
- } else {
- handleField();
- field = line;
- }
- lineNumber += 1;
- }
-
- handleField();
-
- return new MailSimpleParsedHeaders(this.#logger, headers);
- }
-}
-
-export class Mail {
- constructor(public raw: string) {}
-
- toUtf8Bytes(): Uint8Array {
- const utf8Encoder = new TextEncoder();
- return utf8Encoder.encode(this.raw);
- }
-
- toBase64(): string {
- return encodeBase64(this.raw);
- }
-
- startSimpleParse(logger?: Logger) {
- return { sections: () => new MailSimpleParsedSections(logger, this.raw) };
- }
-
- simpleFindAllAddresses(): string[] {
- const re = /,?\<?([a-z0-9_'+\-\.]+\@[a-z0-9_'+\-\.]+)\>?,?/gi;
- return [...this.raw.matchAll(re)].map((m) => m[1]);
- }
-}
-
-export type MailDeliverResultKind = "done" | "fail";
-
-export interface MailDeliverRecipientResult {
- kind: MailDeliverResultKind;
- message: string;
- cause?: unknown;
-}
-
-export class MailDeliverResult {
- message: string = "";
- recipients: Map<string, MailDeliverRecipientResult> = new Map();
-
- constructor(public mail: Mail) {}
-
- hasError(): boolean {
- return (
- this.recipients.size === 0 ||
- this.recipients.values().some((r) => r.kind !== "done")
- );
- }
-
- [Symbol.for("Deno.customInspect")]() {
- return [
- `message: ${this.message}`,
- ...this.recipients
- .entries()
- .map(
- ([recipient, result]) =>
- `${recipient} [${result.kind}]: ${result.message}`,
- ),
- ].join("\n");
- }
-}
-
-export class MailDeliverContext {
- readonly recipients: Set<string> = new Set();
- readonly result;
-
- constructor(
- public readonly logger: Logger,
- public mail: Mail,
- ) {
- this.result = new MailDeliverResult(this.mail);
- }
-}
-
-export interface MailDeliverHook {
- callback(context: MailDeliverContext): Promise<void>;
-}
-
-export abstract class MailDeliverer {
- abstract readonly name: string;
- preHooks: MailDeliverHook[] = [];
- postHooks: MailDeliverHook[] = [];
-
- constructor(protected readonly logger: Logger) {}
-
- protected abstract doDeliver(
- mail: Mail,
- context: MailDeliverContext,
- ): Promise<void>;
-
- async deliverRaw(rawMail: string) {
- return await this.deliver({ mail: new Mail(rawMail) });
- }
-
- async deliver(options: {
- mail: Mail;
- recipients?: string[];
- }): Promise<MailDeliverResult> {
- this.logger.info(`Begin to deliver mail via ${this.name}...`);
-
- const context = new MailDeliverContext(this.logger, options.mail);
- options.recipients?.forEach((r) => context.recipients.add(r));
-
- for (const hook of this.preHooks) {
- await hook.callback(context);
- }
-
- await this.doDeliver(context.mail, context);
-
- for (const hook of this.postHooks) {
- await hook.callback(context);
- }
-
- context.logger.info("Deliver result:\n" + Deno.inspect(context.result));
-
- if (context.result.hasError()) {
- throw new Error("Mail failed to deliver.");
- }
-
- return context.result;
- }
-}
-
-export abstract class SyncMailDeliverer extends MailDeliverer {
- #last: Promise<void> = Promise.resolve();
-
- override async deliver(options: {
- mail: Mail;
- recipients?: string[];
- }): Promise<MailDeliverResult> {
- this.logger.info(
- "The mail deliverer is sync. Wait for last delivering done...",
- );
- await this.#last;
- const result = super.deliver(options);
- this.#last = result.then(
- () => {},
- () => {},
- );
- return result;
- }
-}
-
-export class RecipientFromHeadersHook implements MailDeliverHook {
- constructor(public mailDomain: string) {}
-
- callback(context: MailDeliverContext) {
- if (context.recipients.size !== 0) {
- context.logger.warn(
- "Recipients are already filled. Won't set them with ones in headers.",
- );
- } else {
- context.mail
- .startSimpleParse(context.logger)
- .sections()
- .headers()
- .recipients({
- domain: this.mailDomain,
- })
- .forEach((r) => context.recipients.add(r));
-
- context.logger.info(
- "Recipients found from mail headers: " +
- [...context.recipients].join(", "),
- );
- }
- return Promise.resolve();
- }
-}
-
-export class FallbackRecipientHook implements MailDeliverHook {
- constructor(public fallback: Set<string> = new Set()) {}
-
- callback(context: MailDeliverContext) {
- if (context.recipients.size === 0) {
- context.logger.info(
- "No recipients, fill with fallback: " + [...this.fallback].join(", "),
- );
- this.fallback.forEach((a) => context.recipients.add(a));
- }
- return Promise.resolve();
- }
-}
-
-export class AliasRecipientMailHook implements MailDeliverHook {
- #aliasFile;
-
- constructor(aliasFile: string) {
- this.#aliasFile = aliasFile;
- }
-
- async #parseAliasFile(logger: Logger): Promise<Map<string, string>> {
- const result = new Map();
- if ((await Deno.stat(this.#aliasFile)).isFile) {
- logger.info(`Found recipients alias file: ${this.#aliasFile}.`);
- const text = await Deno.readTextFile(this.#aliasFile);
- const csv = parse(text);
- for (const [real, ...aliases] of csv) {
- aliases.forEach((a) => result.set(a, real));
- }
- }
- return result;
- }
-
- async callback(context: MailDeliverContext) {
- const aliases = await this.#parseAliasFile(context.logger);
- for (const recipient of [...context.recipients]) {
- const realRecipients = aliases.get(recipient);
- if (realRecipients != null) {
- context.logger.info(
- `Recipient alias resolved: ${recipient} => ${realRecipients}.`,
- );
- context.recipients.delete(recipient);
- context.recipients.add(realRecipients);
- }
- }
- }
-}
diff --git a/deno/mail-relay/app.ts b/deno/mail/app.ts
index d96fa1d..2a8c78a 100644
--- a/deno/mail-relay/app.ts
+++ b/deno/mail/app.ts
@@ -1,8 +1,6 @@
import { Hono } from "hono";
import { logger as honoLogger } from "hono/logger";
-import { Logger } from "@crupest/base/log";
-
import {
AliasRecipientMailHook,
FallbackRecipientHook,
@@ -13,20 +11,21 @@ import { DovecotMailDeliverer } from "./dovecot.ts";
import { DumbSmtpServer } from "./dumb-smtp-server.ts";
export function createInbound(
- logger: Logger,
{
fallback,
mailDomain,
aliasFile,
ldaPath,
+ doveadmPath,
}: {
fallback: string[];
mailDomain: string;
aliasFile: string;
ldaPath: string;
+ doveadmPath: string;
},
) {
- const deliverer = new DovecotMailDeliverer(logger, ldaPath);
+ const deliverer = new DovecotMailDeliverer(ldaPath, doveadmPath);
deliverer.preHooks.push(
new RecipientFromHeadersHook(mailDomain),
new FallbackRecipientHook(new Set(fallback)),
@@ -35,42 +34,38 @@ export function createInbound(
return deliverer;
}
-export function createHono(
- logger: Logger,
- outbound: MailDeliverer,
- inbound: MailDeliverer,
-) {
+export function createHono(outbound: MailDeliverer, inbound: MailDeliverer) {
const hono = new Hono();
hono.onError((err, c) => {
- logger.error("Hono handler throws an error.", err);
- return c.json({ msg: "Server error, check its log." }, 500);
+ console.error("Hono handler threw an uncaught error.", err);
+ return c.json({ message: "Server error, check its log." }, 500);
});
hono.use(honoLogger());
hono.post("/send/raw", async (context) => {
const body = await context.req.text();
if (body.trim().length === 0) {
- return context.json({ msg: "Can't send an empty mail." }, 400);
+ return context.json({ message: "Can't send an empty mail." }, 400);
} else {
const result = await outbound.deliverRaw(body);
return context.json({
- awsMessageId: result.awsMessageId,
+ newMessageId: result.newMessageId,
});
}
});
hono.post("/receive/raw", async (context) => {
await inbound.deliverRaw(await context.req.text());
- return context.json({ msg: "Done!" });
+ return context.json({ message: "Done!" });
});
return hono;
}
-export function createSmtp(logger: Logger, outbound: MailDeliverer) {
- return new DumbSmtpServer(logger, outbound);
+export function createSmtp(outbound: MailDeliverer) {
+ return new DumbSmtpServer(outbound);
}
-export async function sendMail(logger: Logger, port: number) {
+export async function sendMail(port: number) {
const decoder = new TextDecoder();
let text = "";
for await (const chunk of Deno.stdin.readable) {
@@ -81,9 +76,8 @@ export async function sendMail(logger: Logger, port: number) {
method: "post",
body: text,
});
- logger.write(Deno.inspect(res), { level: res.ok ? "info" : "error" });
- logger.write(Deno.inspect(await res.text()), {
- level: res.ok ? "info" : "error",
- });
+ const fn = res.ok ? "info" : "error";
+ console[fn](res);
+ console[fn](await res.text());
if (!res.ok) Deno.exit(-1);
}
diff --git a/deno/mail-relay/aws/app.ts b/deno/mail/aws/app.ts
index 685d7a9..7e16488 100644
--- a/deno/mail-relay/aws/app.ts
+++ b/deno/mail/aws/app.ts
@@ -1,23 +1,21 @@
import { join } from "@std/path";
-import { parseArgs } from "@std/cli";
import { z } from "zod";
import { Hono } from "hono";
import { zValidator } from "@hono/zod-validator";
import { FetchHttpHandler } from "@smithy/fetch-http-handler";
+// @ts-types="npm:@types/yargs"
+import yargs from "yargs";
-import { Logger } from "@crupest/base/log";
import { ConfigDefinition, ConfigProvider } from "@crupest/base/config";
import { CronTask } from "@crupest/base/cron";
import { DbService } from "../db.ts";
-import { Mail } from "../mail.ts";
-import {
- AwsMailMessageIdRewriteHook,
- AwsMailMessageIdSaveHook,
-} from "./mail.ts";
+import { createHono, createInbound, createSmtp, sendMail } from "../app.ts";
+import { DovecotMailDeliverer } from "../dovecot.ts";
+import { MailDeliverer } from "../mail.ts";
+import { MessageIdRewriteHook, MessageIdSaveHook } from "../mail.ts";
import { AwsMailDeliverer } from "./deliver.ts";
-import { AwsMailFetcher, AwsS3MailConsumer } from "./fetch.ts";
-import { createInbound, createHono, sendMail, createSmtp } from "../app.ts";
+import { AwsMailFetcher, LiveMailNotFoundError } from "./fetch.ts";
const PREFIX = "crupest-mail-server";
const CONFIG_DEFINITIONS = {
@@ -46,6 +44,10 @@ const CONFIG_DEFINITIONS = {
description: "full path of lda executable",
default: "/dovecot/libexec/dovecot/dovecot-lda",
},
+ doveadmPath: {
+ description: "full path of doveadm executable",
+ default: "/dovecot/bin/doveadm",
+ },
inboundFallback: {
description: "comma separated addresses used as fallback recipients",
default: "",
@@ -93,17 +95,20 @@ function createAwsOptions({
}
function createOutbound(
- logger: Logger,
awsOptions: ReturnType<typeof createAwsOptions>,
db: DbService,
+ local?: DovecotMailDeliverer,
) {
- const deliverer = new AwsMailDeliverer(logger, awsOptions);
+ const deliverer = new AwsMailDeliverer(awsOptions);
deliverer.preHooks.push(
- new AwsMailMessageIdRewriteHook(db.messageIdToAws.bind(db)),
+ new MessageIdRewriteHook(db.messageIdToNew.bind(db)),
);
deliverer.postHooks.push(
- new AwsMailMessageIdSaveHook((original, aws) =>
- db.addMessageIdMap({ message_id: original, aws_message_id: aws }).then(),
+ new MessageIdSaveHook(
+ async (original, new_message_id, context) => {
+ await db.addMessageIdMap({ message_id: original, new_message_id });
+ void local?.saveNewSent(context.logTag, context.mail, original);
+ },
),
);
return deliverer;
@@ -114,15 +119,18 @@ function setupAwsHono(
options: {
path: string;
auth: string;
- callback: (s3Key: string, recipients?: string[]) => Promise<void>;
+ fetcher: AwsMailFetcher;
+ deliverer: MailDeliverer;
},
) {
+ let counter = 1;
+
hono.post(
`/${options.path}`,
async (ctx, next) => {
const auth = ctx.req.header("Authorization");
if (auth !== options.auth) {
- return ctx.json({ msg: "Bad auth!" }, 403);
+ return ctx.json({ message: "Bad auth!" }, 403);
}
await next();
},
@@ -134,19 +142,32 @@ function setupAwsHono(
}),
),
async (ctx) => {
+ const { fetcher, deliverer } = options;
const { key, recipients } = ctx.req.valid("json");
- await options.callback(key, recipients);
- return ctx.json({ msg: "Done!" });
+ try {
+ await fetcher.deliverLiveMail(
+ `[inbound ${counter++}]`,
+ key,
+ deliverer,
+ recipients,
+ );
+ } catch (e) {
+ if (e instanceof LiveMailNotFoundError) {
+ return ctx.json({ message: e.message });
+ }
+ throw e;
+ }
+ return ctx.json({ message: "Done!" });
},
);
}
-function createCron(fetcher: AwsMailFetcher, consumer: AwsS3MailConsumer) {
+function createCron(fetcher: AwsMailFetcher, deliverer: MailDeliverer) {
return new CronTask({
name: "live-mail-recycler",
interval: 6 * 3600 * 1000,
callback: () => {
- return fetcher.recycleLiveMails(consumer);
+ return fetcher.recycleLiveMails(deliverer);
},
startNow: true,
});
@@ -155,84 +176,71 @@ function createCron(fetcher: AwsMailFetcher, consumer: AwsS3MailConsumer) {
function createBaseServices() {
const config = new ConfigProvider(PREFIX, CONFIG_DEFINITIONS);
Deno.mkdirSync(config.get("dataPath"), { recursive: true });
- const logger = new Logger();
- logger.externalLogDir = join(config.get("dataPath"), "log");
- return { config, logger };
+ return { config };
}
function createAwsFetchOnlyServices() {
- const { config, logger } = createBaseServices();
+ const services = createBaseServices();
+ const { config } = services;
+
const awsOptions = createAwsOptions({
user: config.get("awsUser"),
password: config.get("awsPassword"),
region: config.get("awsRegion"),
});
- const fetcher = new AwsMailFetcher(
- logger,
- awsOptions,
- config.get("awsMailBucket"),
- );
- return { config, logger, awsOptions, fetcher };
+ const fetcher = new AwsMailFetcher(awsOptions, config.get("awsMailBucket"));
+
+ return { ...services, awsOptions, fetcher };
}
function createAwsRecycleOnlyServices() {
- const { config, logger, awsOptions, fetcher } = createAwsFetchOnlyServices();
+ const services = createAwsFetchOnlyServices();
+ const { config } = services;
- const inbound = createInbound(logger, {
+ const inbound = createInbound({
fallback: config.getList("inboundFallback"),
ldaPath: config.get("ldaPath"),
+ doveadmPath: config.get("doveadmPath"),
aliasFile: join(config.get("dataPath"), "aliases.csv"),
mailDomain: config.get("mailDomain"),
});
- const recycler = (rawMail: string, _: unknown): Promise<void> =>
- inbound.deliver({ mail: new Mail(rawMail) }).then();
-
- return { config, logger, awsOptions, fetcher, inbound, recycler };
+ return { ...services, inbound };
}
+
function createAwsServices() {
- const { config, logger, inbound, awsOptions, fetcher, recycler } =
- createAwsRecycleOnlyServices();
+ const services = createAwsRecycleOnlyServices();
+ const { config, awsOptions, inbound } = services;
+
const dbService = new DbService(join(config.get("dataPath"), "db.sqlite"));
- const outbound = createOutbound(logger, awsOptions, dbService);
+ const outbound = createOutbound(awsOptions, dbService, inbound);
- return {
- config,
- logger,
- inbound,
- dbService,
- awsOptions,
- fetcher,
- recycler,
- outbound,
- };
+ return { ...services, dbService, outbound };
}
function createServerServices() {
const services = createAwsServices();
- const { logger, config, outbound, inbound, fetcher } = services;
- const smtp = createSmtp(logger, outbound);
+ const { config, outbound, inbound, fetcher } = services;
+
+ const smtp = createSmtp(outbound);
+ const hono = createHono(outbound, inbound);
- const hono = createHono(logger, outbound, inbound);
setupAwsHono(hono, {
path: config.get("awsInboundPath"),
auth: config.get("awsInboundKey"),
- callback: (s3Key, recipients) => {
- return fetcher.consumeS3Mail(s3Key, (rawMail, _) =>
- inbound.deliver({ mail: new Mail(rawMail), recipients }).then(),
- );
- },
+ fetcher,
+ deliverer: inbound,
});
- return {
- ...services,
- smtp,
- hono,
- };
+ return { ...services, smtp, hono };
}
-function serve(cron: boolean = false) {
- const { config, fetcher, recycler, smtp, hono } = createServerServices();
+async function serve(cron: boolean = false) {
+ const { config, fetcher, inbound, smtp, dbService, hono } =
+ createServerServices();
+
+ await dbService.migrate();
+
smtp.serve({
hostname: config.get("smtpHost"),
port: config.getInt("smtpPort"),
@@ -246,55 +254,62 @@ function serve(cron: boolean = false) {
);
if (cron) {
- createCron(fetcher, recycler);
+ createCron(fetcher, inbound);
}
}
async function listLives() {
- const { logger, fetcher } = createAwsFetchOnlyServices();
+ const { fetcher } = createAwsFetchOnlyServices();
const liveMails = await fetcher.listLiveMails();
- logger.info(`Total ${liveMails.length}:`);
- logger.info(liveMails.join("\n"));
+ console.info(`Total ${liveMails.length}:`);
+ if (liveMails.length !== 0) {
+ console.info(liveMails.join("\n"));
+ }
}
async function recycleLives() {
- const { fetcher, recycler } = createAwsRecycleOnlyServices();
- await fetcher.recycleLiveMails(recycler);
+ const { fetcher, inbound } = createAwsRecycleOnlyServices();
+ await fetcher.recycleLiveMails(inbound);
}
if (import.meta.main) {
- const args = parseArgs(Deno.args);
-
- if (args._.length === 0) {
- throw new Error("You must specify a command.");
- }
-
- const command = String(args._[0]);
-
- switch (command) {
- case "sendmail": {
- const { logger, config } = createBaseServices();
- await sendMail(logger, config.getInt("httpPort"));
- break;
- }
- case "list-lives": {
- await listLives();
- break;
- }
- case "recycle-lives": {
- await recycleLives();
- break;
- }
- case "serve": {
- serve();
- break;
- }
- case "real-serve": {
- serve(true);
- break;
- }
- default: {
- throw new Error(command + " is not a valid command.");
- }
- }
+ await yargs(Deno.args)
+ .scriptName("mail")
+ .command({
+ command: "sendmail",
+ describe: "send mail via this server's endpoint",
+ handler: async (_argv) => {
+ const { config } = createBaseServices();
+ await sendMail(config.getInt("httpPort"));
+ },
+ })
+ .command({
+ command: "live",
+ describe: "work with live mails",
+ builder: (builder) => {
+ return builder
+ .command({
+ command: "list",
+ describe: "list live mails",
+ handler: listLives,
+ })
+ .command({
+ command: "recycle",
+ describe: "recycle all live mails",
+ handler: recycleLives,
+ })
+ .demandCommand(1, "One command must be specified.");
+ },
+ handler: () => {},
+ })
+ .command({
+ command: "serve",
+ describe: "start the http and smtp servers",
+ builder: (builder) => builder.option("real", { type: "boolean" }),
+ handler: (argv) => serve(argv.real),
+ })
+ .demandCommand(1, "One command must be specified.")
+ .help()
+ .strict()
+ .parse();
}
diff --git a/deno/mail/aws/deliver.ts b/deno/mail/aws/deliver.ts
new file mode 100644
index 0000000..37a871d
--- /dev/null
+++ b/deno/mail/aws/deliver.ts
@@ -0,0 +1,57 @@
+import {
+ SendEmailCommand,
+ SESv2Client,
+ SESv2ClientConfig,
+} from "@aws-sdk/client-sesv2";
+
+import { Mail, MailDeliverContext, MailDeliverer } from "../mail.ts";
+
+export class AwsMailDeliverer extends MailDeliverer {
+ readonly name = "aws";
+ readonly #aws;
+ readonly #ses;
+
+ constructor(aws: SESv2ClientConfig) {
+ super(true);
+ this.#aws = aws;
+ this.#ses = new SESv2Client(aws);
+ }
+
+ protected override async doDeliver(
+ mail: Mail,
+ context: MailDeliverContext,
+ ): Promise<void> {
+ try {
+ const sendCommand = new SendEmailCommand({
+ Content: {
+ Raw: { Data: mail.toUtf8Bytes() },
+ },
+ });
+
+ console.info(context.logTag, "Calling aws send-email api...");
+ const res = await this.#ses.send(sendCommand);
+ if (res.MessageId == null) {
+ console.warn(
+ context.logTag,
+ "AWS send-email returned null message id.",
+ );
+ } else {
+ context.result.newMessageId =
+ `${res.MessageId}@${this.#aws.region}.amazonses.com`;
+ }
+
+ context.result.messageForSmtp =
+ `AWS Message ID: ${context.result.newMessageId}`;
+ context.result.recipients.set("*", {
+ kind: "success",
+ message: `Succeeded to call aws send-email api.`,
+ });
+ } catch (cause) {
+ context.result.recipients.set("*", {
+ kind: "failure",
+ message: "A JS error was thrown when calling aws send-email." + cause,
+ cause,
+ });
+ }
+ }
+}
diff --git a/deno/mail-relay/aws/fetch.ts b/deno/mail/aws/fetch.ts
index ef1ba5f..2154972 100644
--- a/deno/mail-relay/aws/fetch.ts
+++ b/deno/mail/aws/fetch.ts
@@ -3,14 +3,17 @@ import {
DeleteObjectCommand,
GetObjectCommand,
ListObjectsV2Command,
+ NoSuchKey,
S3Client,
S3ClientConfig,
} from "@aws-sdk/client-s3";
-import { toFileNameString } from "@crupest/base/date";
-import { Logger } from "@crupest/base/log";
+import { DateUtils } from "@crupest/base";
import { Mail } from "../mail.ts";
+import { MailDeliverer } from "../mail.ts";
+
+export class LiveMailNotFoundError extends Error {}
async function s3MoveObject(
client: S3Client,
@@ -34,27 +37,18 @@ async function s3MoveObject(
const AWS_SES_S3_SETUP_TAG = "AMAZON_SES_SETUP_NOTIFICATION";
-export type AwsS3MailConsumer = (
- rawMail: string,
- s3Key: string,
-) => Promise<void>;
-
export class AwsMailFetcher {
readonly #livePrefix = "mail/live/";
readonly #archivePrefix = "mail/archive/";
- readonly #logger;
readonly #s3;
readonly #bucket;
- constructor(logger: Logger, aws: S3ClientConfig, bucket: string) {
- this.#logger = logger;
+ constructor(aws: S3ClientConfig, bucket: string) {
this.#s3 = new S3Client(aws);
this.#bucket = bucket;
}
async listLiveMails(): Promise<string[]> {
- this.#logger.info("Begin to retrieve live mails.");
-
const listCommand = new ListObjectsV2Command({
Bucket: this.#bucket,
Prefix: this.#livePrefix,
@@ -62,16 +56,14 @@ export class AwsMailFetcher {
const res = await this.#s3.send(listCommand);
if (res.Contents == null) {
- this.#logger.warn("Listing live mails in S3 returns null Content.");
+ console.warn("S3 API returned null Content.");
return [];
}
const result: string[] = [];
for (const object of res.Contents) {
if (object.Key == null) {
- this.#logger.warn(
- "Listing live mails in S3 returns an object with no Key.",
- );
+ console.warn("S3 API returned null Key.");
continue;
}
@@ -82,50 +74,63 @@ export class AwsMailFetcher {
return result;
}
- async consumeS3Mail(s3Key: string, consumer: AwsS3MailConsumer) {
- this.#logger.info(`Begin to consume s3 mail ${s3Key} ...`);
-
- this.#logger.info(`Fetching s3 mail ${s3Key}...`);
+ async deliverLiveMail(
+ logTag: string,
+ s3Key: string,
+ deliverer: MailDeliverer,
+ recipients?: string[],
+ ) {
+ console.info(logTag, `Fetching live mail ${s3Key}...`);
const mailPath = `${this.#livePrefix}${s3Key}`;
const command = new GetObjectCommand({
Bucket: this.#bucket,
Key: mailPath,
});
- const res = await this.#s3.send(command);
- if (res.Body == null) {
- throw new Error("S3 mail returns a null body.");
- }
+ let rawMail;
- const rawMail = await res.Body.transformToString();
- this.#logger.info(`Done fetching s3 mail ${s3Key}.`);
+ try {
+ const res = await this.#s3.send(command);
+ if (res.Body == null) {
+ throw new Error("S3 API returns a null body.");
+ }
+ rawMail = await res.Body.transformToString();
+ } catch (cause) {
+ if (cause instanceof NoSuchKey) {
+ const message =
+ `Live mail ${s3Key} is not found. Perhaps already delivered?`;
+ console.error(message, cause);
+ throw new LiveMailNotFoundError(message);
+ }
+ throw cause;
+ }
- this.#logger.info(`Calling consumer...`);
- await consumer(rawMail, s3Key);
- this.#logger.info(`Done consuming s3 mail ${s3Key}.`);
+ const mail = new Mail(rawMail);
+ await deliverer.deliver({ mail, recipients });
- const date = new Mail(rawMail)
- .startSimpleParse(this.#logger)
- .sections()
- .headers()
- .date();
- const dateString =
- date != null ? toFileNameString(date, true) : "invalid-date";
+ const { date } = new Mail(rawMail).parsed;
+ const dateString = date != null
+ ? DateUtils.toFileNameString(date, true)
+ : "invalid-date";
const newPath = `${this.#archivePrefix}${dateString}/${s3Key}`;
- this.#logger.info(`Archiving s3 mail ${s3Key} to ${newPath}...`);
+ console.info(logTag, `Archiving live mail ${s3Key} to ${newPath}...`);
await s3MoveObject(this.#s3, this.#bucket, mailPath, newPath);
- this.#logger.info(`Done archiving s3 mail ${s3Key}.`);
- this.#logger.info(`Done consuming s3 mail ${s3Key}.`);
+ console.info(logTag, `Done deliver live mail ${s3Key}.`);
}
- async recycleLiveMails(consumer: AwsS3MailConsumer) {
- this.#logger.info("Begin to recycle live mails...");
+ async recycleLiveMails(deliverer: MailDeliverer) {
+ console.info("Begin to recycle live mails...");
const mails = await this.listLiveMails();
- this.#logger.info(`Found ${mails.length} live mails`);
+ console.info(`Found ${mails.length} live mails`);
+ let counter = 1;
for (const s3Key of mails) {
- await this.consumeS3Mail(s3Key, consumer);
+ await this.deliverLiveMail(
+ `[${counter++}/${mails.length}]`,
+ s3Key,
+ deliverer,
+ );
}
}
}
diff --git a/deno/mail-relay/db.test.ts b/deno/mail/db.test.ts
index 60035c4..8a9ad27 100644
--- a/deno/mail-relay/db.test.ts
+++ b/deno/mail/db.test.ts
@@ -6,17 +6,17 @@ import { DbService } from "./db.ts";
describe("DbService", () => {
const mockRow = {
message_id: "mock-message-id@mock.mock",
- aws_message_id: "mock-aws-message-id@mock.mock",
+ new_message_id: "mock-new-message-id@mock.mock",
};
it("works", async () => {
const db = new DbService(":memory:");
await db.migrate();
await db.addMessageIdMap(mockRow);
- expect(await db.messageIdToAws(mockRow.message_id)).toBe(
- mockRow.aws_message_id,
+ expect(await db.messageIdToNew(mockRow.message_id)).toBe(
+ mockRow.new_message_id,
);
- expect(await db.messageIdFromAws(mockRow.aws_message_id)).toBe(
+ expect(await db.messageIdFromNew(mockRow.new_message_id)).toBe(
mockRow.message_id,
);
});
diff --git a/deno/mail-relay/db.ts b/deno/mail/db.ts
index 807ecf6..e41f762 100644
--- a/deno/mail-relay/db.ts
+++ b/deno/mail/db.ts
@@ -1,5 +1,3 @@
-// spellchecker: words kysely insertable updateable introspector
-
import {
Generated,
Insertable,
@@ -55,14 +53,14 @@ class SqliteDatabaseAdapter implements SqliteDatabase {
export class DbError extends Error {}
-interface AwsMessageIdMapTable {
+interface MessageIdMapTable {
id: Generated<number>;
message_id: string;
- aws_message_id: string;
+ new_message_id: string;
}
interface Database {
- aws_message_id_map: AwsMessageIdMapTable;
+ message_id_map: MessageIdMapTable;
}
const migrations: Record<string, Migration> = {
@@ -70,16 +68,16 @@ const migrations: Record<string, Migration> = {
// deno-lint-ignore no-explicit-any
async up(db: Kysely<any>): Promise<void> {
await db.schema
- .createTable("aws_message_id_map")
+ .createTable("message_id_map")
.addColumn("id", "integer", (col) => col.primaryKey().autoIncrement())
.addColumn("message_id", "text", (col) => col.notNull().unique())
- .addColumn("aws_message_id", "text", (col) => col.notNull().unique())
+ .addColumn("new_message_id", "text", (col) => col.notNull().unique())
.execute();
- for (const column of ["message_id", "aws_message_id"]) {
+ for (const column of ["message_id", "new_message_id"]) {
await db.schema
- .createIndex(`aws_message_id_map_${column}`)
- .on("aws_message_id_map")
+ .createIndex(`message_id_map_${column}`)
+ .on("message_id_map")
.column(column)
.execute();
}
@@ -87,7 +85,7 @@ const migrations: Record<string, Migration> = {
// deno-lint-ignore no-explicit-any
async down(db: Kysely<any>): Promise<void> {
- await db.schema.dropTable("aws_message_id_map").execute();
+ await db.schema.dropTable("message_id_map").execute();
},
},
};
@@ -119,28 +117,28 @@ export class DbService {
}
async addMessageIdMap(
- mail: Insertable<AwsMessageIdMapTable>,
+ mail: Insertable<MessageIdMapTable>,
): Promise<number> {
const inserted = await this.#kysely
- .insertInto("aws_message_id_map")
+ .insertInto("message_id_map")
.values(mail)
.executeTakeFirstOrThrow();
return Number(inserted.insertId!);
}
- async messageIdToAws(messageId: string): Promise<string | null> {
+ async messageIdToNew(messageId: string): Promise<string | null> {
const row = await this.#kysely
- .selectFrom("aws_message_id_map")
+ .selectFrom("message_id_map")
.where("message_id", "=", messageId)
- .select("aws_message_id")
+ .select("new_message_id")
.executeTakeFirst();
- return row?.aws_message_id ?? null;
+ return row?.new_message_id ?? null;
}
- async messageIdFromAws(awsMessageId: string): Promise<string | null> {
+ async messageIdFromNew(newMessageId: string): Promise<string | null> {
const row = await this.#kysely
- .selectFrom("aws_message_id_map")
- .where("aws_message_id", "=", awsMessageId)
+ .selectFrom("message_id_map")
+ .where("new_message_id", "=", newMessageId)
.select("message_id")
.executeTakeFirst();
return row?.message_id ?? null;
diff --git a/deno/mail-relay/deno.json b/deno/mail/deno.json
index 9105747..86a8999 100644
--- a/deno/mail-relay/deno.json
+++ b/deno/mail/deno.json
@@ -2,7 +2,7 @@
"version": "0.1.0",
"tasks": {
"run": "deno run -A aws/app.ts",
- "compile": "deno compile -o out/crupest-relay -A aws/app.ts"
+ "compile": "deno compile -o out/crupest-mail -A aws/app.ts"
},
"imports": {
"@aws-sdk/client-s3": "npm:@aws-sdk/client-s3@^3.821.0",
diff --git a/deno/mail/dovecot.ts b/deno/mail/dovecot.ts
new file mode 100644
index 0000000..c0d56a2
--- /dev/null
+++ b/deno/mail/dovecot.ts
@@ -0,0 +1,219 @@
+import { Mail, MailDeliverContext, MailDeliverer } from "./mail.ts";
+
+// https://doc.dovecot.org/main/core/man/dovecot-lda.1.html
+const ldaExitCodeMessageMap = new Map<number, string>();
+ldaExitCodeMessageMap.set(67, "recipient user not known");
+ldaExitCodeMessageMap.set(75, "temporary error");
+
+type CommandResult = {
+ kind: "exit";
+ status: Deno.CommandStatus;
+ logMessage: string;
+} | { kind: "throw"; cause: unknown; logMessage: string };
+
+async function runCommand(
+ bin: string,
+ options: {
+ logTag: string;
+ args: string[];
+ stdin?: Uint8Array;
+ suppressStartLog?: boolean;
+ suppressResultLog?: boolean;
+ errorCodeMessageMap?: Map<number, string>;
+ },
+): Promise<CommandResult> {
+ const { logTag, args, stdin, suppressResultLog, errorCodeMessageMap } =
+ options;
+
+ if (options.suppressResultLog !== true) {
+ console.info(logTag, `Run external command ${bin} ${args.join(" ")}`);
+ }
+
+ try {
+ // Create and spawn process.
+ const command = new Deno.Command(bin, {
+ args,
+ stdin: stdin == null ? "null" : "piped",
+ });
+ const process = command.spawn();
+
+ // Write stdin if any.
+ if (stdin != null) {
+ const writer = process.stdin.getWriter();
+ await writer.write(stdin);
+ writer.close();
+ }
+
+ // Wait for process to exit.
+ const status = await process.status;
+
+ // Build log message string.
+ let message = `External command exited with code ${status.code}`;
+ if (status.signal != null) message += ` (signal: ${status.signal})`;
+ if (errorCodeMessageMap != null && errorCodeMessageMap.has(status.code)) {
+ message += `, ${errorCodeMessageMap.get(status.code)}`;
+ }
+ message += ".";
+ if (suppressResultLog !== true) console.log(logTag, message);
+
+ // Return result.
+ return {
+ kind: "exit",
+ status,
+ logMessage: message,
+ };
+ } catch (cause) {
+ const message = `A JS error was thrown when invoking external command:`;
+ if (suppressResultLog !== true) console.log(logTag, message);
+ return { kind: "throw", cause, logMessage: message + " " + cause };
+ }
+}
+
+export class DovecotMailDeliverer extends MailDeliverer {
+ readonly name = "dovecot";
+ readonly #ldaPath;
+ readonly #doveadmPath;
+
+ constructor(
+ ldaPath: string,
+ doveadmPath: string,
+ ) {
+ super(false);
+ this.#ldaPath = ldaPath;
+ this.#doveadmPath = doveadmPath;
+ }
+
+ protected override async doDeliver(
+ mail: Mail,
+ context: MailDeliverContext,
+ ): Promise<void> {
+ const utf8Bytes = mail.toUtf8Bytes();
+
+ const recipients = [...context.recipients];
+
+ if (recipients.length === 0) {
+ throw new Error(
+ "Failed to deliver to dovecot, no recipients are specified.",
+ );
+ }
+
+ for (const recipient of recipients) {
+ const result = await runCommand(
+ this.#ldaPath,
+ {
+ logTag: context.logTag,
+ args: ["-d", recipient],
+ stdin: utf8Bytes,
+ suppressResultLog: true,
+ errorCodeMessageMap: ldaExitCodeMessageMap,
+ },
+ );
+
+ if (result.kind === "exit" && result.status.success) {
+ context.result.recipients.set(recipient, {
+ kind: "success",
+ message: result.logMessage,
+ });
+ } else {
+ context.result.recipients.set(recipient, {
+ kind: "failure",
+ message: result.logMessage,
+ });
+ }
+ }
+ }
+
+ #queryArgs(mailbox: string, messageId: string) {
+ return ["mailbox", mailbox, "header", "Message-ID", `<${messageId}>`];
+ }
+
+ async #deleteMail(
+ logTag: string,
+ user: string,
+ mailbox: string,
+ messageId: string,
+ noLog?: boolean,
+ ): Promise<void> {
+ await runCommand(this.#doveadmPath, {
+ logTag,
+ args: ["expunge", "-u", user, ...this.#queryArgs(mailbox, messageId)],
+ suppressStartLog: noLog,
+ suppressResultLog: noLog,
+ });
+ }
+
+ async #saveMail(
+ logTag: string,
+ user: string,
+ mailbox: string,
+ mail: Uint8Array,
+ ) {
+ await runCommand(this.#doveadmPath, {
+ logTag,
+ args: ["save", "-u", user, "-m", mailbox],
+ stdin: mail,
+ });
+ }
+
+ async #markAsRead(
+ logTag: string,
+ user: string,
+ mailbox: string,
+ messageId: string,
+ ) {
+ await runCommand(this.#doveadmPath, {
+ logTag,
+ args: [
+ "flags",
+ "add",
+ "-u",
+ user,
+ "\\Seen",
+ ...this.#queryArgs(mailbox, messageId),
+ ],
+ });
+ }
+
+ async saveNewSent(logTag: string, mail: Mail, messageIdToDelete: string) {
+ console.info(logTag, "Save sent mail and delete ones with old message id.");
+
+ // Try to get from and recipients from headers.
+ const { messageId, from, recipients } = mail.parsed;
+
+ if (from == null) {
+ console.warn(
+ logTag,
+ "Failed to get sender (from) in headers, skip saving.",
+ );
+ return;
+ }
+
+ if (recipients.includes(from)) {
+ // So the mail should lie in the Inbox.
+ console.info(
+ logTag,
+ "One recipient of the mail is the sender itself, skip saving.",
+ );
+ return;
+ }
+
+ await this.#saveMail(logTag, from, "Sent", mail.toUtf8Bytes());
+ if (messageId != null) {
+ await this.#markAsRead(logTag, from, "Sent", messageId);
+ } else {
+ console.warn(
+ "Message id of the mail is not found, skip marking as read.",
+ );
+ }
+
+ console.info(
+ logTag,
+ "Schedule deletion of old mails (no logging) at 5,15,30,60 seconds later.",
+ );
+ [5, 15, 30, 60].forEach((seconds) =>
+ setTimeout(() => {
+ void this.#deleteMail(logTag, from, "Sent", messageIdToDelete, true);
+ }, 1000 * seconds)
+ );
+ }
+}
diff --git a/deno/mail-relay/dumb-smtp-server.ts b/deno/mail/dumb-smtp-server.ts
index 1a1090a..c3ebf5d 100644
--- a/deno/mail-relay/dumb-smtp-server.ts
+++ b/deno/mail/dumb-smtp-server.ts
@@ -1,4 +1,3 @@
-import { Logger } from "@crupest/base/log";
import { MailDeliverer } from "./mail.ts";
const CRLF = "\r\n";
@@ -13,33 +12,30 @@ function createResponses(host: string, port: number | string) {
RCPT: "250 2.1.5 Recipient OK",
DATA: "354 Start mail input; end with <CRLF>.<CRLF>",
QUIT: `211 2.0.0 ${serverName} closing connection`,
+ ACTIVE_CLOSE: "421 4.7.0 Please open a new connection to send more emails",
INVALID: "500 5.5.1 Error: command not recognized",
} as const;
}
-const LOG_TAG = "[dumb-smtp]";
-
export class DumbSmtpServer {
- #logger;
#deliverer;
- #responses: ReturnType<typeof createResponses> = createResponses(
- "invalid",
- "invalid",
- );
- constructor(logger: Logger, deliverer: MailDeliverer) {
- this.#logger = logger;
+ constructor(deliverer: MailDeliverer) {
this.#deliverer = deliverer;
}
- async #handleConnection(conn: Deno.Conn) {
+ async #handleConnection(
+ logTag: string,
+ conn: Deno.Conn,
+ responses: ReturnType<typeof createResponses>,
+ ) {
using disposeStack = new DisposableStack();
disposeStack.defer(() => {
- this.#logger.tagInfo(LOG_TAG, "Close session's tcp connection.");
+ console.info(logTag, "Close tcp connection.");
conn.close();
});
- this.#logger.tagInfo(LOG_TAG, "New session's tcp connection established.");
+ console.info(logTag, "New tcp connection established.");
const writer = conn.writable.getWriter();
disposeStack.defer(() => writer.releaseLock());
@@ -49,14 +45,14 @@ export class DumbSmtpServer {
const [decoder, encoder] = [new TextDecoder(), new TextEncoder()];
const decode = (data: Uint8Array) => decoder.decode(data);
const send = async (s: string) => {
- this.#logger.tagInfo(LOG_TAG, "Send line: " + s);
+ console.info(logTag, "Send line:", s);
await writer.write(encoder.encode(s + CRLF));
};
let buffer: string = "";
let rawMail: string | null = null;
- await send(this.#responses["READY"]);
+ await send(responses["READY"]);
while (true) {
const { value, done } = await reader.read();
@@ -72,45 +68,37 @@ export class DumbSmtpServer {
buffer = buffer.slice(eolPos + CRLF.length);
if (rawMail == null) {
- this.#logger.tagInfo(LOG_TAG, "Received line: " + line);
+ console.info(logTag, "Received line:", line);
const upperLine = line.toUpperCase();
if (upperLine.startsWith("EHLO") || upperLine.startsWith("HELO")) {
- await send(this.#responses["EHLO"]);
+ await send(responses["EHLO"]);
} else if (upperLine.startsWith("MAIL FROM:")) {
- await send(this.#responses["MAIL"]);
+ await send(responses["MAIL"]);
} else if (upperLine.startsWith("RCPT TO:")) {
- await send(this.#responses["RCPT"]);
+ await send(responses["RCPT"]);
} else if (upperLine === "DATA") {
- await send(this.#responses["DATA"]);
- this.#logger.tagInfo(LOG_TAG, "Begin to receive mail data...");
+ await send(responses["DATA"]);
+ console.info(logTag, "Begin to receive mail data...");
rawMail = "";
} else if (upperLine === "QUIT") {
- await send(this.#responses["QUIT"]);
+ await send(responses["QUIT"]);
return;
} else {
- this.#logger.tagWarn(
- LOG_TAG,
- "Unrecognized command from client: " + line,
- );
- await send(this.#responses["INVALID"]);
+ await send(responses["INVALID"]);
return;
}
} else {
if (line === ".") {
try {
- this.#logger.tagInfo(
- LOG_TAG,
- "Mail data Received, begin to relay...",
- );
- const { message } = await this.#deliverer.deliverRaw(rawMail);
- await send(`250 2.6.0 ${message}`);
+ console.info(logTag, "Mail data received, begin to relay...");
+ const result = await this.#deliverer.deliverRaw(rawMail);
+ await send(`250 2.6.0 ${result.generateMessageForSmtp()}`);
rawMail = null;
- this.#logger.tagInfo(LOG_TAG, "Relay succeeded.");
} catch (err) {
- this.#logger.tagError(LOG_TAG, "Relay failed.", err);
+ console.error(logTag, "Relay failed.", err);
await send("554 5.3.0 Error: check server log");
- return;
}
+ await send(responses["ACTIVE_CLOSE"]);
} else {
const dataLine = line.startsWith("..") ? line.slice(1) : line;
rawMail += dataLine + CRLF;
@@ -122,21 +110,19 @@ export class DumbSmtpServer {
async serve(options: { hostname: string; port: number }) {
const listener = Deno.listen(options);
- this.#responses = createResponses(options.hostname, options.port);
- this.#logger.tagInfo(
- LOG_TAG,
- `Dumb SMTP server starts to listen on ${this.#responses.serverName}.`,
+ const responses = createResponses(options.hostname, options.port);
+ console.info(
+ `Dumb SMTP server starts to listen on ${responses.serverName}.`,
);
+ let counter = 1;
+
for await (const conn of listener) {
+ const logTag = `[outbound ${counter++}]`;
try {
- await this.#handleConnection(conn);
+ await this.#handleConnection(logTag, conn, responses);
} catch (cause) {
- this.#logger.tagError(
- LOG_TAG,
- "Tcp connection throws an error.",
- cause,
- );
+ console.error(logTag, "A JS error was thrown by handler:", cause);
}
}
}
diff --git a/deno/mail/mail-parsing.ts b/deno/mail/mail-parsing.ts
new file mode 100644
index 0000000..8e9697d
--- /dev/null
+++ b/deno/mail/mail-parsing.ts
@@ -0,0 +1,144 @@
+import emailAddresses from "email-addresses";
+
+class MailParsingError extends Error {}
+
+function parseHeaderSection(section: string) {
+ const headers = [] as [key: string, value: string][];
+
+ let field: string | null = null;
+ let lineNumber = 1;
+
+ const handleField = () => {
+ if (field == null) return;
+ const sepPos = field.indexOf(":");
+ if (sepPos === -1) {
+ throw new MailParsingError(
+ `Expect ':' in the header field line: ${field}`,
+ );
+ }
+ headers.push([field.slice(0, sepPos).trim(), field.slice(sepPos + 1)]);
+ field = null;
+ };
+
+ for (const line of section.trimEnd().split(/\r?\n|\r/)) {
+ if (line.match(/^\s/)) {
+ if (field == null) {
+ throw new MailParsingError("Header section starts with a space.");
+ }
+ field += line;
+ } else {
+ handleField();
+ field = line;
+ }
+ lineNumber += 1;
+ }
+
+ handleField();
+
+ return headers;
+}
+
+function findFirst(fields: readonly [string, string][], key: string) {
+ for (const [k, v] of fields) {
+ if (key.toLowerCase() === k.toLowerCase()) return v;
+ }
+ return undefined;
+}
+
+function findMessageId(fields: readonly [string, string][]) {
+ const messageIdField = findFirst(fields, "message-id");
+ if (messageIdField == null) return undefined;
+
+ const match = messageIdField.match(/\<(.*?)\>/);
+ if (match != null) {
+ return match[1];
+ } else {
+ console.warn(`Invalid syntax in header 'message-id': ${messageIdField}`);
+ return undefined;
+ }
+}
+
+function findDate(fields: readonly [string, string][]) {
+ const dateField = findFirst(fields, "date");
+ if (dateField == null) return undefined;
+
+ const date = new Date(dateField);
+ if (isNaN(date.getTime())) {
+ console.warn(`Invalid date string in header 'date': ${dateField}`);
+ return undefined;
+ }
+ return date;
+}
+
+function findFrom(fields: readonly [string, string][]) {
+ const fromField = findFirst(fields, "from");
+ if (fromField == null) return undefined;
+
+ const addr = emailAddresses.parseOneAddress(fromField);
+ return addr?.type === "mailbox" ? addr.address : undefined;
+}
+
+function findRecipients(fields: readonly [string, string][]) {
+ const headers = ["to", "cc", "bcc", "x-original-to"];
+ const recipients = new Set<string>();
+ for (const [key, value] of fields) {
+ if (headers.includes(key.toLowerCase())) {
+ emailAddresses
+ .parseAddressList(value)
+ ?.flatMap((a) => (a.type === "mailbox" ? a : a.addresses))
+ ?.forEach(({ address }) => recipients.add(address));
+ }
+ }
+ return recipients;
+}
+
+function parseSections(raw: string) {
+ const twoEolMatch = raw.match(/(\r?\n)(\r?\n)/);
+ if (twoEolMatch == null) {
+ throw new MailParsingError(
+ "No header/body section separator (2 successive EOLs) found.",
+ );
+ }
+
+ const [eol, sep] = [twoEolMatch[1], twoEolMatch[2]];
+
+ if (eol !== sep) {
+ console.warn("Different EOLs (\\r\\n, \\n) found.");
+ }
+
+ return {
+ header: raw.slice(0, twoEolMatch.index!),
+ body: raw.slice(twoEolMatch.index! + eol.length + sep.length),
+ eol,
+ sep,
+ };
+}
+
+export type ParsedMail = Readonly<{
+ header: string;
+ body: string;
+ sep: string;
+ eol: string;
+ headers: readonly [string, string][];
+ messageId: string | undefined;
+ date: Date | undefined;
+ from: string | undefined;
+ recipients: readonly string[];
+}>;
+
+export function simpleParseMail(raw: string): ParsedMail {
+ const sections = Object.freeze(parseSections(raw));
+ const headers = Object.freeze(parseHeaderSection(sections.header));
+ const messageId = findMessageId(headers);
+ const date = findDate(headers);
+ const from = findFrom(headers);
+ const recipients = Object.freeze([...findRecipients(headers)]);
+ return Object.freeze({
+ ...sections,
+ headers,
+ messageId,
+ date,
+ from,
+ recipients,
+ });
+}
diff --git a/deno/mail-relay/mail.test.ts b/deno/mail/mail.test.ts
index 09cf8eb..a8204be 100644
--- a/deno/mail-relay/mail.test.ts
+++ b/deno/mail/mail.test.ts
@@ -1,8 +1,6 @@
import { describe, it } from "@std/testing/bdd";
import { expect, fn } from "@std/expect";
-import { Logger } from "@crupest/base/log";
-
import { Mail, MailDeliverContext, MailDeliverer } from "./mail.ts";
const mockDate = "Fri, 02 May 2025 08:33:02 +0000";
@@ -53,7 +51,7 @@ const mockToAddresses = [
describe("Mail", () => {
it("simple parse", () => {
- const parsed = new Mail(mockMailStr).startSimpleParse().sections();
+ const { parsed } = new Mail(mockMailStr);
expect(parsed.header).toEqual(mockHeaderStr);
expect(parsed.body).toEqual(mockBodyStr);
expect(parsed.sep).toBe("\n");
@@ -61,37 +59,29 @@ describe("Mail", () => {
});
it("simple parse crlf", () => {
- const parsed = new Mail(mockCrlfMailStr).startSimpleParse().sections();
+ const { parsed } = new Mail(mockCrlfMailStr);
expect(parsed.sep).toBe("\r\n");
expect(parsed.eol).toBe("\r\n");
});
it("simple parse date", () => {
expect(
- new Mail(mockMailStr).startSimpleParse().sections().headers().date(),
+ new Mail(mockMailStr).parsed.date,
).toEqual(new Date(mockDate));
});
it("simple parse headers", () => {
expect(
- new Mail(mockMailStr).startSimpleParse().sections().headers().fields,
+ new Mail(mockMailStr).parsed.headers,
).toEqual(mockHeaders.map((h) => [h[0], " " + h[1].replaceAll("\n", "")]));
});
it("parse recipients", () => {
const mail = new Mail(mockMailStr);
- expect([
- ...mail.startSimpleParse().sections().headers().recipients(),
- ]).toEqual([...mockToAddresses, mockCcAddress]);
- expect([
- ...mail.startSimpleParse().sections().headers().recipients({
- domain: "example.com",
- }),
- ]).toEqual(
- [...mockToAddresses, mockCcAddress].filter((a) =>
- a.endsWith("example.com"),
- ),
- );
+ expect([...mail.parsed.recipients]).toEqual([
+ ...mockToAddresses,
+ mockCcAddress,
+ ]);
});
it("find all addresses", () => {
@@ -115,11 +105,14 @@ describe("MailDeliverer", () => {
class MockMailDeliverer extends MailDeliverer {
name = "mock";
override doDeliver = fn((_: Mail, ctx: MailDeliverContext) => {
- ctx.result.recipients.set("*", { kind: "done", message: "success" });
+ ctx.result.recipients.set("*", {
+ kind: "success",
+ message: "success message",
+ });
return Promise.resolve();
}) as MailDeliverer["doDeliver"];
}
- const mockDeliverer = new MockMailDeliverer(new Logger());
+ const mockDeliverer = new MockMailDeliverer(false);
it("deliver success", async () => {
await mockDeliverer.deliverRaw(mockMailStr);
diff --git a/deno/mail/mail.ts b/deno/mail/mail.ts
new file mode 100644
index 0000000..b88ce2b
--- /dev/null
+++ b/deno/mail/mail.ts
@@ -0,0 +1,304 @@
+import { encodeBase64 } from "@std/encoding/base64";
+import { parse } from "@std/csv/parse";
+
+import { StringUtils } from "@crupest/base";
+
+import { simpleParseMail } from "./mail-parsing.ts";
+
+export class Mail {
+ #raw;
+ #parsed;
+
+ constructor(raw: string) {
+ this.#raw = raw;
+ this.#parsed = simpleParseMail(raw);
+ }
+
+ get raw() {
+ return this.#raw;
+ }
+
+ set raw(value) {
+ this.#raw = value;
+ this.#parsed = simpleParseMail(value);
+ }
+
+ get parsed() {
+ return this.#parsed;
+ }
+
+ toUtf8Bytes(): Uint8Array {
+ const utf8Encoder = new TextEncoder();
+ return utf8Encoder.encode(this.raw);
+ }
+
+ toBase64(): string {
+ return encodeBase64(this.raw);
+ }
+
+ simpleFindAllAddresses(): string[] {
+ const re = /,?\<?([a-z0-9_'+\-\.]+\@[a-z0-9_'+\-\.]+)\>?,?/gi;
+ return [...this.raw.matchAll(re)].map((m) => m[1]);
+ }
+}
+
+export interface MailDeliverRecipientResult {
+ kind: "success" | "failure";
+ message?: string;
+ cause?: unknown;
+}
+
+export class MailDeliverResult {
+ message?: string;
+ messageForSmtp?: string;
+ newMessageId?: string;
+
+ recipients = new Map<string, MailDeliverRecipientResult>();
+ constructor(public mail: Mail) {}
+
+ get hasFailure() {
+ return this.recipients.values().some((v) => v.kind !== "success");
+ }
+
+ generateLogMessage(prefix: string) {
+ const lines = [];
+ if (this.message != null) lines.push(`${prefix} message: ${this.message}`);
+ if (this.messageForSmtp != null) {
+ lines.push(`${prefix} smtpMessage: ${this.messageForSmtp}`);
+ }
+ for (const [name, result] of this.recipients.entries()) {
+ const { kind, message } = result;
+ lines.push(`${prefix} (${name}): ${kind} ${message}`);
+ }
+ return lines.join("\n");
+ }
+
+ generateMessageForSmtp(): string {
+ if (this.messageForSmtp != null) return this.messageForSmtp;
+ return `2.0.0 OK${
+ StringUtils.prependNonEmpty(this.newMessageId)
+ } Message accepted for delivery`;
+ }
+}
+
+export class MailDeliverContext {
+ readonly recipients: Set<string> = new Set();
+ readonly result;
+
+ constructor(public logTag: string, public mail: Mail) {
+ this.result = new MailDeliverResult(this.mail);
+ }
+}
+
+export interface MailDeliverHook {
+ callback(context: MailDeliverContext): Promise<void>;
+}
+
+export abstract class MailDeliverer {
+ #counter = 1;
+ #last?: Promise<void>;
+
+ abstract name: string;
+ preHooks: MailDeliverHook[] = [];
+ postHooks: MailDeliverHook[] = [];
+
+ constructor(public sync: boolean) {}
+
+ protected abstract doDeliver(
+ mail: Mail,
+ context: MailDeliverContext,
+ ): Promise<void>;
+
+ async deliverRaw(rawMail: string) {
+ return await this.deliver({ mail: new Mail(rawMail) });
+ }
+
+ async #deliverCore(context: MailDeliverContext) {
+ for (const hook of this.preHooks) {
+ await hook.callback(context);
+ }
+
+ await this.doDeliver(context.mail, context);
+
+ for (const hook of this.postHooks) {
+ await hook.callback(context);
+ }
+ }
+
+ async deliver(options: {
+ mail: Mail;
+ recipients?: string[];
+ logTag?: string;
+ }): Promise<MailDeliverResult> {
+ const logTag = options.logTag ?? `[${this.name} ${this.#counter}]`;
+ this.#counter++;
+
+ if (this.#last != null) {
+ console.info(logTag, "Wait for last delivering done...");
+ await this.#last;
+ }
+
+ const context = new MailDeliverContext(
+ logTag,
+ options.mail,
+ );
+ options.recipients?.forEach((r) => context.recipients.add(r));
+
+ console.info(context.logTag, "Begin to deliver mail...");
+
+ const deliverPromise = this.#deliverCore(context);
+
+ if (this.sync) {
+ this.#last = deliverPromise.then(() => {}, () => {});
+ }
+
+ await deliverPromise;
+ this.#last = undefined;
+
+ console.info(context.logTag, "Deliver result:");
+ console.info(context.result.generateLogMessage(context.logTag));
+
+ if (context.result.hasFailure) {
+ throw new Error("Failed to deliver to some recipients.");
+ }
+
+ return context.result;
+ }
+}
+
+export class RecipientFromHeadersHook implements MailDeliverHook {
+ constructor(public mailDomain: string) {}
+
+ callback(context: MailDeliverContext) {
+ if (context.recipients.size !== 0) {
+ console.warn(
+ context.logTag,
+ "Recipients are already filled, skip inferring from headers.",
+ );
+ } else {
+ [...context.mail.parsed.recipients].filter((r) =>
+ r.endsWith("@" + this.mailDomain)
+ ).forEach((r) => context.recipients.add(r));
+
+ console.info(
+ context.logTag,
+ "Use recipients inferred from mail headers:",
+ [...context.recipients].join(", "),
+ );
+ }
+ return Promise.resolve();
+ }
+}
+
+export class FallbackRecipientHook implements MailDeliverHook {
+ constructor(public fallback: Set<string> = new Set()) {}
+
+ callback(context: MailDeliverContext) {
+ if (context.recipients.size === 0) {
+ console.info(
+ context.logTag,
+ "Use fallback recipients:" + [...this.fallback].join(", "),
+ );
+ this.fallback.forEach((a) => context.recipients.add(a));
+ }
+ return Promise.resolve();
+ }
+}
+
+export class AliasRecipientMailHook implements MailDeliverHook {
+ #aliasFile;
+
+ constructor(aliasFile: string) {
+ this.#aliasFile = aliasFile;
+ }
+
+ async #parseAliasFile(logTag: string): Promise<Map<string, string>> {
+ const result = new Map();
+ if ((await Deno.stat(this.#aliasFile)).isFile) {
+ const text = await Deno.readTextFile(this.#aliasFile);
+ const csv = parse(text);
+ for (const [real, ...aliases] of csv) {
+ aliases.forEach((a) => result.set(a, real));
+ }
+ } else {
+ console.warn(
+ logTag,
+ `Recipient alias file ${this.#aliasFile} is not found.`,
+ );
+ }
+ return result;
+ }
+
+ async callback(context: MailDeliverContext) {
+ const aliases = await this.#parseAliasFile(context.logTag);
+ for (const recipient of [...context.recipients]) {
+ const realRecipients = aliases.get(recipient);
+ if (realRecipients != null) {
+ console.info(
+ context.logTag,
+ `Recipient alias resolved: ${recipient} => ${realRecipients}.`,
+ );
+ context.recipients.delete(recipient);
+ context.recipients.add(realRecipients);
+ }
+ }
+ }
+}
+
+export class MessageIdRewriteHook implements MailDeliverHook {
+ readonly #lookup;
+
+ constructor(lookup: (origin: string) => Promise<string | null>) {
+ this.#lookup = lookup;
+ }
+
+ async callback(context: MailDeliverContext): Promise<void> {
+ const addresses = context.mail.simpleFindAllAddresses();
+ for (const address of addresses) {
+ const newMessageId = await this.#lookup(address);
+ if (newMessageId != null && newMessageId.length !== 0) {
+ console.info(
+ context.logTag,
+ `Rewrite address-line string in mail: ${address} => ${newMessageId}.`,
+ );
+ context.mail.raw = context.mail.raw.replaceAll(address, newMessageId);
+ }
+ }
+ }
+}
+
+export class MessageIdSaveHook implements MailDeliverHook {
+ readonly #record;
+
+ constructor(
+ record: (
+ original: string,
+ newMessageId: string,
+ context: MailDeliverContext,
+ ) => Promise<void>,
+ ) {
+ this.#record = record;
+ }
+
+ async callback(context: MailDeliverContext): Promise<void> {
+ const { messageId } = context.mail.parsed;
+ if (messageId == null) {
+ console.warn(
+ context.logTag,
+ "Original mail doesn't have message id, skip saving message id map.",
+ );
+ return;
+ }
+ if (context.result.newMessageId != null) {
+ console.info(
+ context.logTag,
+ `Save message id map: ${messageId} => ${context.result.newMessageId}.`,
+ );
+ context.mail.raw = context.mail.raw.replaceAll(
+ messageId,
+ context.result.newMessageId,
+ );
+ await this.#record(messageId, context.result.newMessageId, context);
+ }
+ }
+}
diff --git a/deno/service-manager/deno.json b/deno/service-manager/deno.json
deleted file mode 100644
index 9f30853..0000000
--- a/deno/service-manager/deno.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "version": "0.1.0",
- "tasks": {
- "run": "deno run -A main.ts",
- "compile": "deno compile -o out/manage -A main.ts"
- },
- "imports": {
- "@std/dotenv": "jsr:@std/dotenv@^0.225.5",
- "@std/fs": "jsr:@std/fs@^1.0.18",
- "mustache": "npm:mustache@^4.2.0"
- }
-}
diff --git a/deno/service-manager/main.ts b/deno/service-manager/main.ts
deleted file mode 100644
index 93f4c1b..0000000
--- a/deno/service-manager/main.ts
+++ /dev/null
@@ -1,39 +0,0 @@
-import { parseArgs } from "@std/cli";
-import { loadVariables, TemplateDir } from "./template.ts";
-import { join } from "@std/path";
-
-if (import.meta.main) {
- const args = parseArgs(Deno.args, {
- string: ["project-dir"],
- boolean: ["no-dry-run"],
- });
-
- if (args._.length === 0) {
- throw new Error("You must specify a command.");
- }
-
- const projectDir = args["project-dir"];
- if (projectDir == null) {
- throw new Error("You must specify project-dir.");
- }
-
- const command = String(args._[0]);
-
- switch (command) {
- case "gen-tmpl":
- new TemplateDir(
- join(projectDir, "services/templates"),
- ).generateWithVariableFiles(
- [
- join(projectDir, "data/config"),
- join(projectDir, "services/config.template"),
- ],
- args["no-dry-run"] === true
- ? join(projectDir, "services/generated")
- : undefined,
- );
- break;
- default:
- throw new Error(command + " is not a valid command.");
- }
-}
diff --git a/deno/service-manager/template.ts b/deno/service-manager/template.ts
deleted file mode 100644
index 0b043a1..0000000
--- a/deno/service-manager/template.ts
+++ /dev/null
@@ -1,122 +0,0 @@
-import { dirname, join, relative } from "@std/path";
-import { copySync, existsSync, walkSync } from "@std/fs";
-import { parse } from "@std/dotenv";
-import { distinct } from "@std/collections";
-// @ts-types="npm:@types/mustache"
-import Mustache from "mustache";
-
-Mustache.tags = ["@@", "@@"];
-Mustache.escape = (value) => String(value);
-
-function getVariableKeys(original: string): string[] {
- return distinct(
- Mustache.parse(original)
- .filter(function (v) {
- return v[0] === "name";
- })
- .map(function (v) {
- return v[1];
- }),
- );
-}
-
-export function loadVariables(files: string[]): Record<string, string> {
- const vars: Record<string, string> = {};
- for (const file of files) {
- const text = Deno.readTextFileSync(file);
- for (const [key, valueText] of Object.entries(parse(text))) {
- getVariableKeys(valueText).forEach((name) => {
- if (!(name in vars)) {
- throw new Error(
- `Variable ${name} is not defined yet, perhaps due to typos or wrong order.`,
- );
- }
- });
- vars[key] = Mustache.render(valueText, vars);
- }
- }
- return vars;
-}
-
-const TEMPLATE_FILE_EXT = ".template";
-
-export class TemplateDir {
- templates: { path: string; ext: string; text: string; vars: string[] }[] = [];
- plains: { path: string }[] = [];
-
- constructor(public dir: string) {
- console.log("Scanning template dir:");
- Array.from(
- walkSync(dir, { includeDirs: false, followSymlinks: true }),
- ).forEach(({ path }) => {
- path = relative(this.dir, path);
- if (path.endsWith(TEMPLATE_FILE_EXT)) {
- console.log(` (template) ${path}`);
- const text = Deno.readTextFileSync(join(dir, path));
- this.templates.push({
- path,
- ext: TEMPLATE_FILE_EXT,
- text,
- vars: getVariableKeys(text),
- });
- } else {
- console.log(` (plain) ${path}`);
- this.plains.push({ path });
- }
- });
- console.log("Done scanning template dir.");
- }
-
- allNeededVars() {
- return distinct(this.templates.flatMap((t) => t.vars));
- }
-
- generate(vars: Record<string, string>, generatedDir?: string) {
- console.log(
- `Generating, template dir: ${this.dir}, generated dir: ${generatedDir ?? "[dry-run]"}:`,
- );
-
- const undefinedVars = this.allNeededVars().filter((v) => !(v in vars));
- if (undefinedVars.length !== 0) {
- throw new Error(
- `Needed variables are not defined: ${undefinedVars.join(", ")}`,
- );
- }
-
- if (generatedDir != null) {
- if (existsSync(generatedDir)) {
- console.log(` delete old generated dir ${generatedDir}`);
- Deno.removeSync(generatedDir, { recursive: true });
- }
-
- for (const file of this.plains) {
- const [source, destination] = [
- join(this.dir, file.path),
- join(generatedDir, file.path),
- ];
- console.log(` copy ${source} to ${destination} ...`);
- Deno.mkdirSync(dirname(destination), { recursive: true });
- copySync(source, destination);
- }
- for (const file of this.templates) {
- const [source, destination] = [
- join(this.dir, file.path),
- join(generatedDir, file.path.slice(0, -file.ext.length)),
- ];
- console.log(` generate ${source} to ${destination} ...`);
- const rendered = Mustache.render(file.text, vars);
- Deno.mkdirSync(dirname(destination), { recursive: true });
- Deno.writeTextFileSync(destination, rendered);
- }
- }
- console.log(`Done generating.`);
- }
-
- generateWithVariableFiles(varFiles: string[], generatedDir?: string) {
- console.log("Scanning defined vars:");
- const vars = loadVariables(varFiles);
- Object.keys(vars).forEach((name) => console.log(` ${name}`));
- console.log("Done scanning defined vars.");
- this.generate(vars, generatedDir);
- }
-}
diff --git a/deno/tools/deno.json b/deno/tools/deno.json
index 3597182..355046a 100644
--- a/deno/tools/deno.json
+++ b/deno/tools/deno.json
@@ -3,5 +3,6 @@
"tasks": {
},
"imports": {
+ "mustache": "npm:mustache@^4.2.0"
}
}
diff --git a/deno/tools/gen-geosite-rules.ts b/deno/tools/geosite.ts
index c59d34f..3aabec2 100644
--- a/deno/tools/gen-geosite-rules.ts
+++ b/deno/tools/geosite.ts
@@ -1,7 +1,7 @@
-const PROXY_NAME = "node-select"
-const ATTR = "cn"
+const ATTR = "cn";
const REPO_NAME = "domain-list-community";
-const URL = "https://github.com/v2fly/domain-list-community/archive/refs/heads/master.zip"
+const URL =
+ "https://github.com/v2fly/domain-list-community/archive/refs/heads/master.zip";
const SITES = [
"github",
"google",
@@ -39,9 +39,9 @@ const SITES = [
"ieee",
"sci-hub",
"libgen",
-]
+];
-const prefixes = ["include", "domain", "keyword", "full", "regexp"] as const
+const prefixes = ["include", "domain", "keyword", "full", "regexp"] as const;
interface Rule {
kind: (typeof prefixes)[number];
@@ -52,20 +52,20 @@ interface Rule {
type FileProvider = (name: string) => string;
function extract(starts: string[], provider: FileProvider): Rule[] {
-function parseLine(line: string): Rule {
- let kind = prefixes.find((p) => line.startsWith(p + ":"));
- if (kind != null) {
- line = line.slice(line.indexOf(":") + 1);
- } else {
- kind = "domain";
+ function parseLine(line: string): Rule {
+ let kind = prefixes.find((p) => line.startsWith(p + ":"));
+ if (kind != null) {
+ line = line.slice(line.indexOf(":") + 1);
+ } else {
+ kind = "domain";
+ }
+ const segs = line.split("@");
+ return {
+ kind,
+ value: segs[0].trim(),
+ attrs: [...segs.slice(1)].map((s) => s.trim()),
+ };
}
- const segs = line.split("@");
- return {
- kind,
- value: segs[0].trim(),
- attrs: [...segs.slice(1)].map((s) => s.trim()),
- };
-}
function parse(text: string): Rule[] {
return text
@@ -76,10 +76,10 @@ function parseLine(line: string): Rule {
.map((l) => parseLine(l));
}
- const visited = [] as string[]
- const rules = [] as Rule[]
+ const visited = [] as string[];
+ const rules = [] as Rule[];
- function add(name :string) {
+ function add(name: string) {
const text = provider(name);
for (const rule of parse(text)) {
if (rule.kind === "include") {
@@ -100,25 +100,25 @@ function parseLine(line: string): Rule {
add(start);
}
- return rules
+ return rules;
}
function toNewFormat(rules: Rule[], attr: string): [string, string] {
function toLine(rule: Rule) {
const prefixMap = {
- "domain": "DOMAIN-SUFFIX",
- "full": "DOMAIN",
- "keyword": "DOMAIN-KEYWORD",
- "regexp": "DOMAIN-REGEX",
+ domain: "DOMAIN-SUFFIX",
+ full: "DOMAIN",
+ keyword: "DOMAIN-KEYWORD",
+ regexp: "DOMAIN-REGEX",
} as const;
if (rule.kind === "include") {
- throw new Error("Include rule not parsed.")
+ throw new Error("Include rule not parsed.");
}
- return `${prefixMap[rule.kind]},${rule.value}`
+ return `${prefixMap[rule.kind]},${rule.value}`;
}
function toLines(rules: Rule[]) {
- return rules.map(r => toLine(r)).join("\n")
+ return rules.map((r) => toLine(r)).join("\n");
}
const has: Rule[] = [];
@@ -128,7 +128,6 @@ function toNewFormat(rules: Rule[], attr: string): [string, string] {
return [toLines(has), toLines(notHas)];
}
-
if (import.meta.main) {
const tmpDir = Deno.makeTempDirSync({ prefix: "geosite-rules-" });
console.log("Work dir is ", tmpDir);
@@ -150,12 +149,13 @@ if (import.meta.main) {
const provider = (name: string) =>
Deno.readTextFileSync(dataDir + "/" + name);
- const rules = extract(SITES, provider)
- const [has, notHas] = toNewFormat(rules, ATTR)
- const hasFile = tmpDir + "/has-rule"
- const notHasFile = tmpDir + "/not-has-rule"
- console.log("Write result to: " + hasFile + " , " + notHasFile)
- Deno.writeTextFileSync(hasFile, has)
- Deno.writeTextFileSync(notHasFile, notHas)
+ const rules = extract(SITES, provider);
+ const [has, notHas] = toNewFormat(rules, ATTR);
+ const resultDir = tmpDir + "/result";
+ Deno.mkdirSync(resultDir);
+ const hasFile = resultDir + "/has-rule";
+ const notHasFile = resultDir + "/not-has-rule";
+ console.log("Write result to: " + hasFile + " , " + notHasFile);
+ Deno.writeTextFileSync(hasFile, has);
+ Deno.writeTextFileSync(notHasFile, notHas);
}
-
diff --git a/deno/tools/main.ts b/deno/tools/main.ts
new file mode 100644
index 0000000..897350c
--- /dev/null
+++ b/deno/tools/main.ts
@@ -0,0 +1,14 @@
+import yargs, { DEMAND_COMMAND_MESSAGE } from "./yargs.ts";
+import vm from "./vm.ts";
+import service from "./service.ts";
+
+if (import.meta.main) {
+ await yargs(Deno.args)
+ .scriptName("crupest")
+ .command(vm)
+ .command(service)
+ .demandCommand(1, DEMAND_COMMAND_MESSAGE)
+ .help()
+ .strict()
+ .parse();
+}
diff --git a/deno/tools/service.ts b/deno/tools/service.ts
new file mode 100644
index 0000000..bd4d22c
--- /dev/null
+++ b/deno/tools/service.ts
@@ -0,0 +1,180 @@
+import { dirname, join, relative } from "@std/path";
+import { copySync, existsSync, walkSync } from "@std/fs";
+import { distinct } from "@std/collections";
+// @ts-types="npm:@types/mustache"
+import Mustache from "mustache";
+
+import { defineYargsModule, DEMAND_COMMAND_MESSAGE } from "./yargs.ts";
+
+const MUSTACHE_RENDER_OPTIONS: Mustache.RenderOptions = {
+ tags: ["@@", "@@"],
+ escape: (value: unknown) => String(value),
+};
+
+function mustacheParse(template: string) {
+ return Mustache.parse(template, MUSTACHE_RENDER_OPTIONS.tags);
+}
+
+function mustacheRender(template: string, view: Record<string, string>) {
+ return Mustache.render(template, view, {}, MUSTACHE_RENDER_OPTIONS);
+}
+
+function getVariableKeysOfTemplate(template: string): string[] {
+ return distinct(
+ mustacheParse(template)
+ .filter((v) => v[0] === "name")
+ .map((v) => v[1]),
+ );
+}
+
+function loadTemplatedConfigFiles(
+ files: string[],
+): Record<string, string> {
+ console.log("Scan config files ...");
+ const config: Record<string, string> = {};
+ for (const file of files) {
+ console.log(` from file ${file}`);
+ const text = Deno.readTextFileSync(file);
+ let lineNumber = 0;
+ for (const rawLine of text.split("\n")) {
+ lineNumber++;
+ const line = rawLine.trim();
+ if (line.length === 0) continue;
+ if (line.startsWith("#")) continue;
+ const equalSymbolIndex = line.indexOf("=");
+ if (equalSymbolIndex === -1) {
+ throw new Error(`Line ${lineNumber} of ${file} is invalid.`);
+ }
+ const [key, valueText] = [
+ line.slice(0, equalSymbolIndex).trim(),
+ line.slice(equalSymbolIndex + 1).trim(),
+ ];
+ console.log(` (${key in config ? "override" : "new"}) ${key}`);
+ getVariableKeysOfTemplate(valueText).forEach((name) => {
+ if (!(name in config)) {
+ throw new Error(
+ `Variable ${name} is not defined yet, perhaps due to typos or wrong order.`,
+ );
+ }
+ });
+ config[key] = mustacheRender(valueText, config);
+ }
+ }
+ return config;
+}
+
+const TEMPLATE_FILE_EXT = ".template";
+
+class TemplateDir {
+ templates: { path: string; ext: string; text: string; vars: string[] }[] = [];
+ plains: { path: string }[] = [];
+
+ constructor(public dir: string) {
+ console.log(`Scan template dir ${dir} ...`);
+ Array.from(
+ walkSync(dir, { includeDirs: false, followSymlinks: true }),
+ ).forEach(({ path }) => {
+ path = relative(this.dir, path);
+ if (path.endsWith(TEMPLATE_FILE_EXT)) {
+ console.log(` (template) ${path}`);
+ const text = Deno.readTextFileSync(join(dir, path));
+ this.templates.push({
+ path,
+ ext: TEMPLATE_FILE_EXT,
+ text,
+ vars: getVariableKeysOfTemplate(text),
+ });
+ } else {
+ console.log(` (plain) ${path}`);
+ this.plains.push({ path });
+ }
+ });
+ }
+
+ allNeededVars() {
+ return distinct(this.templates.flatMap((t) => t.vars));
+ }
+
+ generate(vars: Record<string, string>, generatedDir?: string) {
+ console.log(
+ `Generate to dir ${generatedDir ?? "[dry-run]"} ...`,
+ );
+
+ const undefinedVars = this.allNeededVars().filter((v) => !(v in vars));
+ if (undefinedVars.length !== 0) {
+ throw new Error(
+ `Needed variables are not defined: ${undefinedVars.join(", ")}`,
+ );
+ }
+
+ if (generatedDir != null) {
+ if (existsSync(generatedDir)) {
+ console.log(` delete old generated dir`);
+ Deno.removeSync(generatedDir, { recursive: true });
+ }
+
+ for (const file of this.plains) {
+ const [source, destination] = [
+ join(this.dir, file.path),
+ join(generatedDir, file.path),
+ ];
+ console.log(` copy ${file.path}`);
+ Deno.mkdirSync(dirname(destination), { recursive: true });
+ copySync(source, destination);
+ }
+ for (const file of this.templates) {
+ const path = file.path.slice(0, -file.ext.length);
+ const destination = join(generatedDir, path);
+ console.log(` generate ${path}`);
+ const rendered = mustacheRender(file.text, vars);
+ Deno.mkdirSync(dirname(destination), { recursive: true });
+ Deno.writeTextFileSync(destination, rendered);
+ }
+ }
+ }
+}
+
+export default defineYargsModule({
+ command: "service",
+ aliases: ["sv"],
+ describe: "Manage services.",
+ builder: (builder) => {
+ return builder
+ .option("project-dir", {
+ type: "string",
+ })
+ .demandOption("project-dir")
+ .command({
+ command: "gen-tmpl",
+ describe: "Generate files from templates",
+ builder: (builder) => {
+ return builder
+ .option("dry-run", {
+ type: "boolean",
+ default: true,
+ })
+ .strict();
+ },
+ handler: (argv) => {
+ const { projectDir, dryRun } = argv;
+
+ const config = loadTemplatedConfigFiles(
+ [
+ join(projectDir, "data/config"),
+ join(projectDir, "services/config.template"),
+ ],
+ );
+
+ new TemplateDir(
+ join(projectDir, "services/templates"),
+ ).generate(
+ config,
+ dryRun ? undefined : join(projectDir, "services/generated"),
+ );
+ console.log("Done!");
+ },
+ })
+ .demandCommand(1, DEMAND_COMMAND_MESSAGE);
+ },
+ handler: () => {},
+});
diff --git a/deno/tools/vm.ts b/deno/tools/vm.ts
new file mode 100644
index 0000000..b54c0d4
--- /dev/null
+++ b/deno/tools/vm.ts
@@ -0,0 +1,225 @@
+import os from "node:os";
+import { join } from "@std/path";
+import { defineYargsModule, DEMAND_COMMAND_MESSAGE } from "./yargs.ts";
+
+type ArchAliasMap = { [name: string]: string[] };
+const arches = {
+ x86_64: ["x86_64", "amd64"],
+ i386: ["i386", "x86", "i686"],
+} as const satisfies ArchAliasMap;
+type Arch = keyof typeof arches;
+type GeneralArch = (typeof arches)[Arch][number];
+
+function normalizeArch(generalName: GeneralArch): Arch {
+ for (const [name, aliases] of Object.entries(arches as ArchAliasMap)) {
+ if (aliases.includes(generalName)) return name as Arch;
+ }
+ throw Error("Unknown architecture name.");
+}
+
+interface GeneralVmSetup {
+ name?: string[];
+ arch: GeneralArch;
+ cpuNumber?: number;
+ memory?: number;
+ disk: string;
+ usbTablet?: boolean;
+ sshForwardPort?: number;
+ tpm?: boolean;
+ kvm?: boolean;
+}
+
+interface VmSetup {
+ arch: Arch;
+ cpuNumber: number;
+ memory: number;
+ disk: string;
+ usbTablet: boolean;
+ sshForwardPort?: number;
+ tpm: boolean;
+ kvm: boolean;
+}
+
+const VM_DIR = join(os.homedir(), "vms");
+
+function getDiskFilePath(name: string): string {
+ return join(VM_DIR, `${name}.qcow2`);
+}
+
+const MY_VMS: GeneralVmSetup[] = [
+ {
+ name: ["hurd", ...arches.i386.map((a) => `hurd-${a}`)],
+ arch: "i386",
+ disk: getDiskFilePath("hurd-i386"),
+ sshForwardPort: 3222,
+ },
+ {
+ name: [...arches.x86_64.map((a) => `hurd-${a}`)],
+ arch: "x86_64",
+ disk: getDiskFilePath("hurd-x86_64"),
+ sshForwardPort: 3223,
+ },
+ {
+ name: ["win"],
+ arch: "x86_64",
+ cpuNumber: 4,
+ memory: 16,
+ disk: getDiskFilePath("win"),
+ usbTablet: true,
+ tpm: true,
+ },
+];
+
+function normalizeVmSetup(generalSetup: GeneralVmSetup): VmSetup {
+ const { arch, cpuNumber, memory, disk, usbTablet, sshForwardPort, tpm, kvm } =
+ generalSetup;
+
+ const normalizedArch = normalizeArch(arch);
+ const is64 = normalizedArch === "x86_64";
+
+ return {
+ arch: normalizedArch,
+ disk,
+ cpuNumber: cpuNumber ?? 1,
+ memory: memory ?? (is64 ? 8 : 4),
+ usbTablet: usbTablet ?? false,
+ sshForwardPort,
+ tpm: tpm ?? false,
+ kvm: kvm ?? Deno.build.os === "linux",
+ };
+}
+
+function resolveVmSetup(
+ name: string,
+ vms: GeneralVmSetup[],
+): VmSetup | undefined {
+ const setup = vms.find((vm) => vm.name?.includes(name));
+ return setup == null ? undefined : normalizeVmSetup(setup);
+}
+
+const qemuBinPrefix = "qemu-system" as const;
+
+const qemuBinSuffix = {
+ x86_64: "x86_64",
+ i386: "x86_64",
+} as const;
+
+function getQemuBin(arch: Arch): string {
+ return `${qemuBinPrefix}-${qemuBinSuffix[arch]}`;
+}
+
+function getLinuxHostArgs(kvm: boolean): string[] {
+ return kvm ? ["-enable-kvm"] : [];
+}
+
+function getMachineArgs(vm: VmSetup): string[] {
+ const is64 = vm.arch === "x86_64";
+ const machineArgs = is64 ? ["-machine", "q35"] : [];
+ return [...machineArgs, "-smp", String(vm.cpuNumber), "-m", `${vm.memory}G`];
+}
+
+function getDeviceArgs(vm: VmSetup): string[] {
+ const { usbTablet } = vm;
+ return usbTablet ? ["-usb", "-device", "usb-tablet"] : [];
+}
+
+function getNetworkArgs(sshForwardPort?: number): string[] {
+ const args = ["-net", "nic"];
+ if (sshForwardPort != null) {
+ args.push("-net", `user,hostfwd=tcp::${sshForwardPort}-:22`);
+ }
+ return args;
+}
+
+function getDisplayArgs(): string[] {
+ return ["-vga", "vmware"];
+}
+
+function getDiskArgs(disk: string): string[] {
+ return ["-drive", `cache=writeback,file=${disk}`];
+}
+
+function getTpmControlSocketPath(): string {
+ return join(VM_DIR, "tpm2/swtpm-sock");
+}
+
+function getTpmArgs(tpm: boolean): string[] {
+ if (!tpm) return [];
+ return [
+ "-chardev",
+ `socket,id=chrtpm,path=${getTpmControlSocketPath()}`,
+ "-tpmdev",
+ "emulator,id=tpm0,chardev=chrtpm",
+ "-device",
+ "tpm-tis,tpmdev=tpm0",
+ ];
+}
+
+function getTpmPreCommand(): string[] {
+ return [
+ "swtpm",
+ "socket",
+ "--tpm2",
+ "--tpmstate",
+ `dir=${join(VM_DIR, "tpm2")}`,
+ "--ctrl",
+ `type=unixio,path=${getTpmControlSocketPath()}`,
+ ];
+}
+
+function createPreCommands(setup: VmSetup): string[][] {
+ const { tpm } = setup;
+ const result = [];
+ if (tpm) result.push(getTpmPreCommand());
+ return result;
+}
+
+function createQemuArgs(setup: VmSetup): string[] {
+ const { arch, disk, sshForwardPort, tpm } = setup;
+ return [
+ getQemuBin(arch),
+ ...getLinuxHostArgs(setup.kvm),
+ ...getMachineArgs(setup),
+ ...getDeviceArgs(setup),
+ ...getDisplayArgs(),
+ ...getNetworkArgs(sshForwardPort),
+ ...getDiskArgs(disk),
+ ...getTpmArgs(tpm),
+ ];
+}
+
+const gen = defineYargsModule({
+ command: "gen <name>",
+ describe: "generate cli command to run the vm",
+ builder: (builder) => {
+ return builder
+ .positional("name", {
+ describe: "name of the vm to run",
+ type: "string",
+ })
+ .demandOption("name")
+ .strict();
+ },
+ handler: (argv) => {
+ const vm = resolveVmSetup(argv.name, MY_VMS);
+ if (vm == null) {
+ console.error(`No vm called ${argv.name} is found.`);
+ Deno.exit(-1);
+ }
+ const preCommands = createPreCommands(vm);
+ const cli = createQemuArgs(vm);
+ for (const command of preCommands) {
+ console.log(`${command.join(" ")} &`);
+ }
+ console.log(`${cli.join(" ")}`);
+ },
+});
+
+export default defineYargsModule({
+ command: "vm",
+ describe: "Manage (qemu) virtual machines.",
+ builder: (builder) => {
+ return builder.command(gen).demandCommand(1, DEMAND_COMMAND_MESSAGE);
+ },
+ handler: () => {},
+});
diff --git a/deno/tools/yargs.ts b/deno/tools/yargs.ts
new file mode 100644
index 0000000..eaa7803
--- /dev/null
+++ b/deno/tools/yargs.ts
@@ -0,0 +1,12 @@
+// @ts-types="npm:@types/yargs"
+export { default } from "yargs";
+export * from "yargs";
+
+import { CommandModule } from "yargs";
+export function defineYargsModule<T, U>(
+ module: CommandModule<T, U>,
+): CommandModule<T, U> {
+ return module;
+}
+
+export const DEMAND_COMMAND_MESSAGE = "No command is specified";
generated by cgit v1.2.3 (git 2.46.0) at 2025-12-14 20:13:17 +0000