diff options
27 files changed, 552 insertions, 364 deletions
diff --git a/.editorconfig b/.editorconfig index ea11c46..69ce65e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -29,3 +29,6 @@ indent_size = 2 [*.py] profile = black + +[*.lua] +indent_size = 4 diff --git a/deno/base/config.ts b/deno/base/config.ts index 98722f6..8fce1d8 100644 --- a/deno/base/config.ts +++ b/deno/base/config.ts @@ -91,4 +91,3 @@ export class ConfigProvider<K extends string> { .join("\n"); } } - diff --git a/deno/base/log.ts b/deno/base/log.ts index 1a4942d..cc71dfa 100644 --- a/deno/base/log.ts +++ b/deno/base/log.ts @@ -4,43 +4,19 @@ import { toFileNameString } from "./date.ts"; export type LogLevel = "error" | "warn" | "info"; -export interface LogEntry { - content: [unknown, ...unknown[]]; +export interface LogOptions { level?: LogLevel; cause?: unknown; } -export interface LogEntryBuilder { - withLevel(level: LogLevel): LogEntryBuilder; - withCause(cause: unknown): LogEntryBuilder; - setError(error: boolean): LogEntryBuilder; - write(): void; -} - export interface ExternalLogStream extends Disposable { stream: WritableStream; } export class Logger { - #indentSize = 2; + #defaultLevel = "info" as const; #externalLogDir?: string; - #contextStack: { depth: number; level: LogLevel }[] = [ - { depth: 0, level: "info" }, - ]; - - get #context() { - return this.#contextStack.at(-1)!; - } - - get indentSize() { - return this.#indentSize; - } - - set indentSize(value: number) { - this.#indentSize = value; - } - get externalLogDir() { return this.#externalLogDir; } @@ -54,64 +30,37 @@ export class Logger { } } - write(entry: LogEntry): void { - const { content, level, cause } = entry; - const [message, ...rest] = content; - console[level ?? this.#context.level]( - " ".repeat(this.#indentSize * this.#context.depth) + String(message), - ...(cause != null ? [cause, ...rest] : rest), - ); + write(message: string, options?: LogOptions): void { + const logFunction = console[options?.level ?? this.#defaultLevel]; + if (options?.cause != null) { + logFunction.call(console, message, options.cause); + } else { + logFunction.call(console, message); + } } - push(entry: LogEntry): Disposable { - this.write(entry); - this.#contextStack.push({ - depth: this.#context.depth + 1, - level: entry.level ?? this.#context.level, - }); - return { - [Symbol.dispose]: () => { - this.#contextStack.pop(); - }, - }; + info(message: string) { + this.write(message, { level: "info" }); } - info(message: unknown, ...args: unknown[]) { - this.write({ level: "info", content: [message, ...args] }); + tagInfo(tag: string, message: string) { + this.info(tag + " " + message); } - warn(message: unknown, ...args: unknown[]) { - this.write({ level: "warn", content: [message, ...args] }); + warn(message: string) { + this.write(message, { level: "warn" }); } - error(message: unknown, ...args: unknown[]) { - this.write({ level: "error", content: [message, ...args] }); + tagWarn(tag: string, message: string) { + this.warn(tag + " " + message); } - builder(message: unknown, ...args: unknown[]): LogEntryBuilder { - const entry: LogEntry = { - content: [message, ...args], - level: "info", - cause: undefined, - }; - const builder: LogEntryBuilder = { - withCause: (cause) => { - entry.cause = cause; - return builder; - }, - withLevel: (level) => { - entry.level = level; - return builder; - }, - setError: (error) => { - if (error) entry.level = "error"; - return builder; - }, - write: () => { - this.write(entry); - }, - }; - return builder; + error(message: string, cause?: unknown) { + this.write(message, { level: "info", cause }); + } + + tagError(tag: string, message: string, cause?: unknown) { + this.error(tag + " " + message, cause); } async createExternalLogStream( @@ -161,4 +110,3 @@ export class Logger { }; } } - diff --git a/deno/deno.json b/deno/deno.json index 71ad398..55ffcb8 100644 --- a/deno/deno.json +++ b/deno/deno.json @@ -1,5 +1,5 @@ { - "workspace": [ "./base", "./service-manager", "./mail-relay" ], + "workspace": ["./base", "./service-manager", "./mail-relay", "./tools" ], "tasks": { "compile:mail-relay": "deno task --cwd=mail-relay compile", "compile:service-manager": "deno task --cwd=service-manager compile" diff --git a/deno/mail-relay/app.ts b/deno/mail-relay/app.ts index 3cac44b..d96fa1d 100644 --- a/deno/mail-relay/app.ts +++ b/deno/mail-relay/app.ts @@ -43,7 +43,7 @@ export function createHono( const hono = new Hono(); hono.onError((err, c) => { - logger.error(err); + logger.error("Hono handler throws an error.", err); return c.json({ msg: "Server error, check its log." }, 500); }); hono.use(honoLogger()); @@ -81,10 +81,9 @@ export async function sendMail(logger: Logger, port: number) { method: "post", body: text, }); - logger.builder(res).setError(!res.ok).write(); - logger - .builder("Body\n" + (await res.text())) - .setError(!res.ok) - .write(); + logger.write(Deno.inspect(res), { level: res.ok ? "info" : "error" }); + logger.write(Deno.inspect(await res.text()), { + level: res.ok ? "info" : "error", + }); if (!res.ok) Deno.exit(-1); } diff --git a/deno/mail-relay/aws/deliver.ts b/deno/mail-relay/aws/deliver.ts index 3e1f162..9950e37 100644 --- a/deno/mail-relay/aws/deliver.ts +++ b/deno/mail-relay/aws/deliver.ts @@ -45,14 +45,12 @@ export class AwsMailDeliverer extends SyncMailDeliverer { if (res.MessageId == null) { this.#logger.warn("Aws send-email returns no message id."); } else { - context.result.awsMessageId = - `${res.MessageId}@${this.#aws.region}.amazonses.com`; + context.result.awsMessageId = `${res.MessageId}@${this.#aws.region}.amazonses.com`; } context.result.recipients.set("*", { kind: "done", - message: - `Successfully called aws send-email, message id ${context.result.awsMessageId}.`, + message: `Successfully called aws send-email, message id ${context.result.awsMessageId}.`, }); } catch (cause) { context.result.recipients.set("*", { diff --git a/deno/mail-relay/db.ts b/deno/mail-relay/db.ts index 9b05e32..807ecf6 100644 --- a/deno/mail-relay/db.ts +++ b/deno/mail-relay/db.ts @@ -20,19 +20,20 @@ class SqliteStatementAdapter implements SqliteStatement { } all(parameters: readonly unknown[]): unknown[] { - return this.stmt.all(...parameters as sqlite.BindValue[]); + return this.stmt.all(...(parameters as sqlite.BindValue[])); } iterate(parameters: readonly unknown[]): IterableIterator<unknown> { - return this.stmt.iter(...parameters as sqlite.BindValue[]); + return this.stmt.iter(...(parameters as sqlite.BindValue[])); } - run( - parameters: readonly unknown[], - ): { changes: number | bigint; lastInsertRowid: number | bigint } { + run(parameters: readonly unknown[]): { + changes: number | bigint; + lastInsertRowid: number | bigint; + } { const { db } = this.stmt; const totalChangesBefore = db.totalChanges; - const changes = this.stmt.run(...parameters as sqlite.BindValue[]); + const changes = this.stmt.run(...(parameters as sqlite.BindValue[])); return { changes: totalChangesBefore === db.totalChanges ? 0 : changes, lastInsertRowid: db.lastInsertRowId, @@ -52,8 +53,7 @@ class SqliteDatabaseAdapter implements SqliteDatabase { } } -export class DbError extends Error { -} +export class DbError extends Error {} interface AwsMessageIdMapTable { id: Generated<number>; @@ -100,9 +100,9 @@ export class DbService { constructor(public readonly path: string) { this.#db = new sqlite.Database(path); this.#kysely = new Kysely<Database>({ - dialect: new SqliteDialect( - { database: new SqliteDatabaseAdapter(this.#db) }, - ), + dialect: new SqliteDialect({ + database: new SqliteDatabaseAdapter(this.#db), + }), }); this.#migrator = new Migrator({ db: this.#kysely, @@ -121,24 +121,27 @@ export class DbService { async addMessageIdMap( mail: Insertable<AwsMessageIdMapTable>, ): Promise<number> { - const inserted = await this.#kysely.insertInto("aws_message_id_map").values( - mail, - ).executeTakeFirstOrThrow(); + const inserted = await this.#kysely + .insertInto("aws_message_id_map") + .values(mail) + .executeTakeFirstOrThrow(); return Number(inserted.insertId!); } async messageIdToAws(messageId: string): Promise<string | null> { - const row = await this.#kysely.selectFrom("aws_message_id_map").where( - "message_id", - "=", - messageId, - ).select("aws_message_id").executeTakeFirst(); + const row = await this.#kysely + .selectFrom("aws_message_id_map") + .where("message_id", "=", messageId) + .select("aws_message_id") + .executeTakeFirst(); return row?.aws_message_id ?? null; } async messageIdFromAws(awsMessageId: string): Promise<string | null> { - const row = await this.#kysely.selectFrom("aws_message_id_map") - .where("aws_message_id", "=", awsMessageId).select("message_id") + const row = await this.#kysely + .selectFrom("aws_message_id_map") + .where("aws_message_id", "=", awsMessageId) + .select("message_id") .executeTakeFirst(); return row?.message_id ?? null; } diff --git a/deno/mail-relay/dovecot.ts b/deno/mail-relay/dovecot.ts index cb63766..124a82b 100644 --- a/deno/mail-relay/dovecot.ts +++ b/deno/mail-relay/dovecot.ts @@ -2,11 +2,7 @@ import { basename } from "@std/path"; import { Logger } from "@crupest/base/log"; -import { - Mail, - MailDeliverContext, - MailDeliverer, -} from "./mail.ts"; +import { Mail, MailDeliverContext, MailDeliverer } from "./mail.ts"; export class DovecotMailDeliverer extends MailDeliverer { readonly name = "dovecot"; @@ -38,9 +34,7 @@ export class DovecotMailDeliverer extends MailDeliverer { for (const recipient of recipients) { try { const commandArgs = ["-d", recipient]; - this.logger.info( - `Run ${ldaBinName} ${commandArgs.join(" ")}...`, - ); + this.logger.info(`Run ${ldaBinName} ${commandArgs.join(" ")}...`); const ldaCommand = new Deno.Command(ldaPath, { args: commandArgs, @@ -50,9 +44,8 @@ export class DovecotMailDeliverer extends MailDeliverer { }); const ldaProcess = ldaCommand.spawn(); - using logFiles = await this.logger.createExternalLogStreamsForProgram( - ldaBinName, - ); + using logFiles = + await this.logger.createExternalLogStreamsForProgram(ldaBinName); ldaProcess.stdout.pipeTo(logFiles.stdout); ldaProcess.stderr.pipeTo(logFiles.stderr); diff --git a/deno/mail-relay/dumb-smtp-server.ts b/deno/mail-relay/dumb-smtp-server.ts index 66c2f7c..1a1090a 100644 --- a/deno/mail-relay/dumb-smtp-server.ts +++ b/deno/mail-relay/dumb-smtp-server.ts @@ -17,6 +17,8 @@ function createResponses(host: string, port: number | string) { } as const; } +const LOG_TAG = "[dumb-smtp]"; + export class DumbSmtpServer { #logger; #deliverer; @@ -33,9 +35,12 @@ export class DumbSmtpServer { async #handleConnection(conn: Deno.Conn) { using disposeStack = new DisposableStack(); disposeStack.defer(() => { - this.#logger.info("Close smtp session tcp connection."); + this.#logger.tagInfo(LOG_TAG, "Close session's tcp connection."); conn.close(); }); + + this.#logger.tagInfo(LOG_TAG, "New session's tcp connection established."); + const writer = conn.writable.getWriter(); disposeStack.defer(() => writer.releaseLock()); const reader = conn.readable.getReader(); @@ -43,8 +48,10 @@ export class DumbSmtpServer { const [decoder, encoder] = [new TextDecoder(), new TextEncoder()]; const decode = (data: Uint8Array) => decoder.decode(data); - const send = async (s: string) => + const send = async (s: string) => { + this.#logger.tagInfo(LOG_TAG, "Send line: " + s); await writer.write(encoder.encode(s + CRLF)); + }; let buffer: string = ""; let rawMail: string | null = null; @@ -65,7 +72,7 @@ export class DumbSmtpServer { buffer = buffer.slice(eolPos + CRLF.length); if (rawMail == null) { - this.#logger.info("Smtp server received line:", line); + this.#logger.tagInfo(LOG_TAG, "Received line: " + line); const upperLine = line.toUpperCase(); if (upperLine.startsWith("EHLO") || upperLine.startsWith("HELO")) { await send(this.#responses["EHLO"]); @@ -75,26 +82,32 @@ export class DumbSmtpServer { await send(this.#responses["RCPT"]); } else if (upperLine === "DATA") { await send(this.#responses["DATA"]); - this.#logger.info("Begin to receive mail data..."); + this.#logger.tagInfo(LOG_TAG, "Begin to receive mail data..."); rawMail = ""; } else if (upperLine === "QUIT") { await send(this.#responses["QUIT"]); return; } else { - this.#logger.warn("Smtp server command unrecognized:", line); + this.#logger.tagWarn( + LOG_TAG, + "Unrecognized command from client: " + line, + ); await send(this.#responses["INVALID"]); return; } } else { if (line === ".") { try { - this.#logger.info("Done receiving mail data, begin to relay..."); + this.#logger.tagInfo( + LOG_TAG, + "Mail data Received, begin to relay...", + ); const { message } = await this.#deliverer.deliverRaw(rawMail); await send(`250 2.6.0 ${message}`); rawMail = null; - this.#logger.info("Done SMTP mail session."); + this.#logger.tagInfo(LOG_TAG, "Relay succeeded."); } catch (err) { - this.#logger.info(err); + this.#logger.tagError(LOG_TAG, "Relay failed.", err); await send("554 5.3.0 Error: check server log"); return; } @@ -107,21 +120,23 @@ export class DumbSmtpServer { } } - async serve(options: { - hostname: string, - port: number - }) { + async serve(options: { hostname: string; port: number }) { const listener = Deno.listen(options); this.#responses = createResponses(options.hostname, options.port); - this.#logger.info( - `Dumb SMTP server starts running on ${this.#responses.serverName}.`, + this.#logger.tagInfo( + LOG_TAG, + `Dumb SMTP server starts to listen on ${this.#responses.serverName}.`, ); for await (const conn of listener) { try { await this.#handleConnection(conn); } catch (cause) { - this.#logger.error("One smtp connection session throws an error " + cause); + this.#logger.tagError( + LOG_TAG, + "Tcp connection throws an error.", + cause, + ); } } } diff --git a/deno/mail-relay/mail.test.ts b/deno/mail-relay/mail.test.ts index 6f3cd13..09cf8eb 100644 --- a/deno/mail-relay/mail.test.ts +++ b/deno/mail-relay/mail.test.ts @@ -67,33 +67,31 @@ describe("Mail", () => { }); it("simple parse date", () => { - expect(new Mail(mockMailStr).startSimpleParse().sections().headers().date()) - .toEqual(new Date(mockDate)); + expect( + new Mail(mockMailStr).startSimpleParse().sections().headers().date(), + ).toEqual(new Date(mockDate)); }); it("simple parse headers", () => { expect( new Mail(mockMailStr).startSimpleParse().sections().headers().fields, - ).toEqual(mockHeaders.map( - (h) => [h[0], " " + h[1].replaceAll("\n", "")], - )); + ).toEqual(mockHeaders.map((h) => [h[0], " " + h[1].replaceAll("\n", "")])); }); it("parse recipients", () => { const mail = new Mail(mockMailStr); - expect([...mail.startSimpleParse().sections().headers().recipients()]) - .toEqual([ - ...mockToAddresses, - mockCcAddress, - ]); + expect([ + ...mail.startSimpleParse().sections().headers().recipients(), + ]).toEqual([...mockToAddresses, mockCcAddress]); expect([ ...mail.startSimpleParse().sections().headers().recipients({ domain: "example.com", }), - ]).toEqual([ - ...mockToAddresses, - mockCcAddress, - ].filter((a) => a.endsWith("example.com"))); + ]).toEqual( + [...mockToAddresses, mockCcAddress].filter((a) => + a.endsWith("example.com"), + ), + ); }); it("find all addresses", () => { diff --git a/deno/mail-relay/mail.ts b/deno/mail-relay/mail.ts index 8c2e067..12d5972 100644 --- a/deno/mail-relay/mail.ts +++ b/deno/mail-relay/mail.ts @@ -4,12 +4,15 @@ import emailAddresses from "email-addresses"; import { Logger } from "@crupest/base/log"; -class MailSimpleParseError extends Error { } +class MailSimpleParseError extends Error {} class MailSimpleParsedHeaders { - #logger + #logger; - constructor(logger: Logger | undefined, public fields: [key:string, value: string][]) { + constructor( + logger: Logger | undefined, + public fields: [key: string, value: string][], + ) { this.#logger = logger; } @@ -28,7 +31,9 @@ class MailSimpleParsedHeaders { if (match != null) { return match[1]; } else { - this.#logger?.warn("Invalid message-id header of mail: ", messageIdField); + this.#logger?.warn( + "Invalid message-id header of mail: " + messageIdField, + ); return undefined; } } @@ -71,10 +76,10 @@ class MailSimpleParsedSections { eol: string; sep: string; - #logger + #logger; constructor(logger: Logger | undefined, raw: string) { - this.#logger = logger + this.#logger = logger; const twoEolMatch = raw.match(/(\r?\n)(\r?\n)/); if (twoEolMatch == null) { @@ -96,7 +101,7 @@ class MailSimpleParsedSections { } headers(): MailSimpleParsedHeaders { - const headers = [] as [key:string, value: string][]; + const headers = [] as [key: string, value: string][]; let field: string | null = null; let lineNumber = 1; @@ -105,9 +110,7 @@ class MailSimpleParsedSections { if (field == null) return; const sepPos = field.indexOf(":"); if (sepPos === -1) { - throw new MailSimpleParseError( - `No ':' in the header line: ${field}`, - ); + throw new MailSimpleParseError(`No ':' in the header line: ${field}`); } headers.push([field.slice(0, sepPos).trim(), field.slice(sepPos + 1)]); field = null; @@ -149,8 +152,8 @@ export class Mail { } simpleFindAllAddresses(): string[] { - const re = /,?\<?([a-z0-9_'+\-\.]+\@[a-z0-9_'+\-\.]+)\>?,?/ig - return [...this.raw.matchAll(re)].map(m => m[1]) + const re = /,?\<?([a-z0-9_'+\-\.]+\@[a-z0-9_'+\-\.]+)\>?,?/gi; + return [...this.raw.matchAll(re)].map((m) => m[1]); } } @@ -169,16 +172,21 @@ export class MailDeliverResult { constructor(public mail: Mail) {} hasError(): boolean { - return this.recipients.size === 0 || - this.recipients.values().some((r) => r.kind !== "done"); + return ( + this.recipients.size === 0 || + this.recipients.values().some((r) => r.kind !== "done") + ); } [Symbol.for("Deno.customInspect")]() { return [ `message: ${this.message}`, - ...this.recipients.entries().map(([recipient, result]) => - `${recipient} [${result.kind}]: ${result.message}` - ), + ...this.recipients + .entries() + .map( + ([recipient, result]) => + `${recipient} [${result.kind}]: ${result.message}`, + ), ].join("\n"); } } @@ -204,7 +212,7 @@ export abstract class MailDeliverer { preHooks: MailDeliverHook[] = []; postHooks: MailDeliverHook[] = []; - constructor(protected readonly logger: Logger) { } + constructor(protected readonly logger: Logger) {} protected abstract doDeliver( mail: Mail, @@ -234,8 +242,7 @@ export abstract class MailDeliverer { await hook.callback(context); } - context.logger.info("Deliver result:"); - context.logger.info(context.result); + context.logger.info("Deliver result:\n" + Deno.inspect(context.result)); if (context.result.hasError()) { throw new Error("Mail failed to deliver."); @@ -248,13 +255,19 @@ export abstract class MailDeliverer { export abstract class SyncMailDeliverer extends MailDeliverer { #last: Promise<void> = Promise.resolve(); - override async deliver( - options: { mail: Mail; recipients?: string[] }, - ): Promise<MailDeliverResult> { - this.logger.info("The mail deliverer is sync. Wait for last delivering done..."); + override async deliver(options: { + mail: Mail; + recipients?: string[]; + }): Promise<MailDeliverResult> { + this.logger.info( + "The mail deliverer is sync. Wait for last delivering done...", + ); await this.#last; const result = super.deliver(options); - this.#last = result.then(() => {}, () => {}); + this.#last = result.then( + () => {}, + () => {}, + ); return result; } } @@ -278,8 +291,8 @@ export class RecipientFromHeadersHook implements MailDeliverHook { .forEach((r) => context.recipients.add(r)); context.logger.info( - "Recipients found from mail headers: ", - [...context.recipients].join(" "), + "Recipients found from mail headers: " + + [...context.recipients].join(", "), ); } return Promise.resolve(); @@ -292,8 +305,7 @@ export class FallbackRecipientHook implements MailDeliverHook { callback(context: MailDeliverContext) { if (context.recipients.size === 0) { context.logger.info( - "No recipients, fill with fallback: ", - [...this.fallback].join(" "), + "No recipients, fill with fallback: " + [...this.fallback].join(", "), ); this.fallback.forEach((a) => context.recipients.add(a)); } diff --git a/deno/service-manager/deno.json b/deno/service-manager/deno.json index 2ba8394..9f30853 100644 --- a/deno/service-manager/deno.json +++ b/deno/service-manager/deno.json @@ -1,4 +1,3 @@ - { "version": "0.1.0", "tasks": { diff --git a/deno/service-manager/template.ts b/deno/service-manager/template.ts index be6cb60..0b043a1 100644 --- a/deno/service-manager/template.ts +++ b/deno/service-manager/template.ts @@ -1,9 +1,9 @@ import { dirname, join, relative } from "@std/path"; import { copySync, existsSync, walkSync } from "@std/fs"; import { parse } from "@std/dotenv"; -import { distinct } from "@std/collections" +import { distinct } from "@std/collections"; // @ts-types="npm:@types/mustache" -import Mustache from "mustache" +import Mustache from "mustache"; Mustache.tags = ["@@", "@@"]; Mustache.escape = (value) => String(value); @@ -21,7 +21,7 @@ function getVariableKeys(original: string): string[] { } export function loadVariables(files: string[]): Record<string, string> { - const vars: Record<string, string> = {} + const vars: Record<string, string> = {}; for (const file of files) { const text = Deno.readTextFileSync(file); for (const [key, valueText] of Object.entries(parse(text))) { @@ -38,7 +38,7 @@ export function loadVariables(files: string[]): Record<string, string> { return vars; } -const TEMPLATE_FILE_EXT = ".template" +const TEMPLATE_FILE_EXT = ".template"; export class TemplateDir { templates: { path: string; ext: string; text: string; vars: string[] }[] = []; diff --git a/deno/tools/deno.json b/deno/tools/deno.json new file mode 100644 index 0000000..3597182 --- /dev/null +++ b/deno/tools/deno.json @@ -0,0 +1,7 @@ +{ + "version": "0.1.0", + "tasks": { + }, + "imports": { + } +} diff --git a/deno/tools/gen-geosite-rules.ts b/deno/tools/gen-geosite-rules.ts new file mode 100644 index 0000000..c59d34f --- /dev/null +++ b/deno/tools/gen-geosite-rules.ts @@ -0,0 +1,161 @@ +const PROXY_NAME = "node-select" +const ATTR = "cn" +const REPO_NAME = "domain-list-community"; +const URL = "https://github.com/v2fly/domain-list-community/archive/refs/heads/master.zip" +const SITES = [ + "github", + "google", + "youtube", + "twitter", + "facebook", + "discord", + "reddit", + "twitch", + "quora", + "telegram", + "imgur", + "stackexchange", + "onedrive", + "duckduckgo", + "wikimedia", + "gitbook", + "gitlab", + "creativecommons", + "archive", + "matrix", + "tor", + "python", + "ruby", + "rust", + "nodejs", + "npmjs", + "qt", + "docker", + "v2ray", + "homebrew", + "bootstrap", + "heroku", + "vercel", + "ieee", + "sci-hub", + "libgen", +] + +const prefixes = ["include", "domain", "keyword", "full", "regexp"] as const + +interface Rule { + kind: (typeof prefixes)[number]; + value: string; + attrs: string[]; +} + +type FileProvider = (name: string) => string; + +function extract(starts: string[], provider: FileProvider): Rule[] { +function parseLine(line: string): Rule { + let kind = prefixes.find((p) => line.startsWith(p + ":")); + if (kind != null) { + line = line.slice(line.indexOf(":") + 1); + } else { + kind = "domain"; + } + const segs = line.split("@"); + return { + kind, + value: segs[0].trim(), + attrs: [...segs.slice(1)].map((s) => s.trim()), + }; +} + + function parse(text: string): Rule[] { + return text + .replaceAll("\c\n", "\n") + .split("\n") + .map((l) => l.trim()) + .filter((l) => l.length !== 0 && !l.startsWith("#")) + .map((l) => parseLine(l)); + } + + const visited = [] as string[] + const rules = [] as Rule[] + + function add(name :string) { + const text = provider(name); + for (const rule of parse(text)) { + if (rule.kind === "include") { + if (visited.includes(rule.value)) { + console.warn(`circular refs found: ${name} includes ${rule.value}.`); + continue; + } else { + visited.push(rule.value); + add(rule.value); + } + } else { + rules.push(rule); + } + } + } + + for (const start of starts) { + add(start); + } + + return rules +} + +function toNewFormat(rules: Rule[], attr: string): [string, string] { + function toLine(rule: Rule) { + const prefixMap = { + "domain": "DOMAIN-SUFFIX", + "full": "DOMAIN", + "keyword": "DOMAIN-KEYWORD", + "regexp": "DOMAIN-REGEX", + } as const; + if (rule.kind === "include") { + throw new Error("Include rule not parsed.") + } + return `${prefixMap[rule.kind]},${rule.value}` + } + + function toLines(rules: Rule[]) { + return rules.map(r => toLine(r)).join("\n") + } + + const has: Rule[] = []; + const notHas: Rule[] = []; + rules.forEach((r) => (r.attrs.includes(attr) ? has.push(r) : notHas.push(r))); + + return [toLines(has), toLines(notHas)]; +} + + +if (import.meta.main) { + const tmpDir = Deno.makeTempDirSync({ prefix: "geosite-rules-" }); + console.log("Work dir is ", tmpDir); + const zipFilePath = tmpDir + "/repo.zip"; + const res = await fetch(URL); + if (!res.ok) { + throw new Error("Failed to download repo."); + } + Deno.writeFileSync(zipFilePath, await res.bytes()); + const unzip = new Deno.Command("unzip", { + args: ["-q", zipFilePath], + cwd: tmpDir, + }); + if (!(await unzip.spawn().status).success) { + throw new Error("Failed to unzip"); + } + + const dataDir = tmpDir + "/" + REPO_NAME + "-master/data"; + const provider = (name: string) => + Deno.readTextFileSync(dataDir + "/" + name); + + const rules = extract(SITES, provider) + const [has, notHas] = toNewFormat(rules, ATTR) + const hasFile = tmpDir + "/has-rule" + const notHasFile = tmpDir + "/not-has-rule" + console.log("Write result to: " + hasFile + " , " + notHasFile) + Deno.writeTextFileSync(hasFile, has) + Deno.writeTextFileSync(notHasFile, notHas) +} + diff --git a/dictionary.txt b/dictionary.txt index c89911c..03a5f54 100644 --- a/dictionary.txt +++ b/dictionary.txt @@ -23,6 +23,9 @@ esmtp tini healthcheck nspawn +containerd +buildx +fontconfig # vim/nvim nvim diff --git a/store/config/etc/fonts/local.conf b/store/config/etc/fonts/local.conf new file mode 100644 index 0000000..a8dbe2b --- /dev/null +++ b/store/config/etc/fonts/local.conf @@ -0,0 +1,60 @@ +<?xml version="1.0"?> +<!DOCTYPE fontconfig SYSTEM "urn:fontconfig:fonts.dtd"> +<fontconfig> + + <alias> + <family>sans-serif</family> + <prefer> + <family>MiSans</family> + </prefer> + </alias> + + <alias> + <family>serif</family> + <prefer> + <family>MiSans</family> + </prefer> + </alias> + + <alias> + <family>monospace</family> + <prefer> + <family>Maple Mono</family> + </prefer> + </alias> + + <alias> + <family>MiSans</family> + <prefer> + <family>MiSans</family> + <family>Noto Color Emoji</family> + </prefer> + </alias> + + <alias> + <family>Maple Mono</family> + <prefer> + <family>Maple Mono</family> + <family>Maple Mono NF</family> + <family>Maple Mono NF CN</family> + <family>Noto Color Emoji</family> + </prefer> + </alias> + + <alias> + <family>Noto Sans</family> + <prefer> + <family>Noto Sans</family> + <family>Noto Sans CJK SC</family> + </prefer> + </alias> + + <alias> + <family>Noto Serif</family> + <prefer> + <family>Noto Serif</family> + <family>Noto Serif CJK SC</family> + </prefer> + </alias> + +</fontconfig>
\ No newline at end of file diff --git a/store/config/nvim/init.lua b/store/config/nvim/init.lua index d038019..ac6961a 100644 --- a/store/config/nvim/init.lua +++ b/store/config/nvim/init.lua @@ -1,3 +1,5 @@ +vim.crupest = {} + local lazy_path = vim.fn.stdpath("data") .. "/lazy/lazy.nvim" if not vim.uv.fs_stat(lazy_path) then vim.fn.system({ diff --git a/store/config/nvim/lua/setup/plugins/conform.lua b/store/config/nvim/lua/setup/plugins/conform.lua index e14c3f9..57b74a9 100644 --- a/store/config/nvim/lua/setup/plugins/conform.lua +++ b/store/config/nvim/lua/setup/plugins/conform.lua @@ -5,7 +5,6 @@ local function setup() typescript = { "prettierd", "prettier", stop_after_first = true }, javascriptreact = { "prettierd", "prettier", stop_after_first = true }, typescriptreact = { "prettierd", "prettier", stop_after_first = true }, - markdown = { "prettierd", "prettier", stop_after_first = true }, }, default_format_opts = { lsp_format = "fallback", diff --git a/store/config/nvim/lua/setup/plugins/lint.lua b/store/config/nvim/lua/setup/plugins/lint.lua index 928841d..b33db22 100644 --- a/store/config/nvim/lua/setup/plugins/lint.lua +++ b/store/config/nvim/lua/setup/plugins/lint.lua @@ -1,4 +1,6 @@ -local lint = require("lint") +--- spellchecker: words markdownlintrc + +---@alias CruLinter { name: string, config_patterns: string[], filetypes: string[] | nil, fast: boolean } local cspell = { name = "cspell", @@ -31,15 +33,30 @@ local markdownlint = { fast = true, } -local linters = { cspell, markdownlint } - -local linter_names = vim.tbl_map(function(l) return l.name end, linters) +local linters = { cspell = cspell, markdownlint = markdownlint } -local function cru_lint(linter, opt) - opt = opt or {} +---@param linter CruLinter +---@param buf integer +---@return string | nil +local function find_config(linter, buf) + local files = vim.fs.find(linter.config_patterns, { + path = vim.api.nvim_buf_get_name(buf), upward = true }) + if #files ~= 0 then + return files[1]; + end + return nil +end - local buf = opt.buf or 0 +vim.list_extend(require("lint.linters.markdownlint").args, { + "--config", + function() + return find_config(markdownlint, 0); + end +}) +---@param linter CruLinter +---@param buf integer +function vim.crupest.lint(linter, buf) if linter.filetypes then local filetype = vim.api.nvim_get_option_value("filetype", { buf = buf }) if not vim.list_contains(linter.filetypes, filetype) then @@ -47,50 +64,29 @@ local function cru_lint(linter, opt) end end - if 0 ~= #vim.fs.find(linter.config_patterns, { - path = vim.api.nvim_buf_get_name(opt.buf), upward = true }) then - lint.try_lint(linter.name) + if find_config(linter, buf) then + require("lint").try_lint(linter.name) end end -local function cru_lint_one(name, opt) - for _, linter in ipairs(linters) do - if linter.name == name then - cru_lint(linter, opt) - return - end - end - vim.notify("No linter named " .. name .. " is configured.", vim.log.levels.ERROR, {}) -end - -local function cru_lint_all(opt, fast) - for _, linter in ipairs(linters) do +function vim.crupest.lint_all(buf, fast) + for _, linter in pairs(linters) do if not fast or linter.fast then - cru_lint(linter, opt) + vim.crupest.lint(linter, buf) end end end -local function cru_lint_all_fast(opt) - local buf = opt.buf - if vim.api.nvim_get_option_value("buftype", { buf = buf }) == "" then - cru_lint_all(opt, true) - end -end - local function setup() - vim.api.nvim_create_autocmd({ "BufReadPost", "InsertLeave", "TextChanged" }, { callback = cru_lint_all_fast }) - - local function cru_lint_cmd(opt) - if #opt.args == 0 then - cru_lint_all(opt, false) - else - cru_lint_one(opt.args, opt) - end - end - - vim.api.nvim_create_user_command("CruLint", cru_lint_cmd, - { nargs = '?', complete = function() return linter_names end }) + vim.api.nvim_create_autocmd( + { "BufReadPost", "InsertLeave", "TextChanged" }, + { + callback = function(opt) + if vim.api.nvim_get_option_value("buftype", { buf = opt.buf }) == "" then + vim.crupest.lint_all(opt.buf, true) + end + end + }) end return { diff --git a/www/.markdownlint.json b/www/.markdownlint.json index 2811a59..12bc65e 100644 --- a/www/.markdownlint.json +++ b/www/.markdownlint.json @@ -1,6 +1,5 @@ { "$schema": "https://raw.githubusercontent.com/DavidAnson/markdownlint/main/schema/markdownlint-config-schema.json", "default": true, - "MD013": false, - "MD014": false + "MD013": false } diff --git a/www/content/notes/cheat-sheet.md b/www/content/notes/cheat-sheet.md index 77e337d..56bc92a 100644 --- a/www/content/notes/cheat-sheet.md +++ b/www/content/notes/cheat-sheet.md @@ -1,18 +1,24 @@ --- title: "Cheat Sheet" date: 2025-04-01T23:09:53+08:00 -lastmod: 2025-04-01T23:09:53+08:00 +lastmod: 2025-06-12T01:09:39+08:00 --- +{{< mono >}} + +goto: [Hurd Cheat Sheet (in a separated page)](/notes/hurd/cheat-sheet) + +{{< /mono >}} + ## GRUB Update GRUB after `grub` package is updated. Replace `/boot` with your mount point of the EFI partition in `--efi-directory=/boot`. Replace `GRUB` with your bootloader id in `--bootloader-id=GRUB`. -```bash-session -# grub-install --target=x86_64-efi --efi-directory=/boot --bootloader-id=GRUB -# grub-mkconfig -o /boot/grub/grub.cfg +```sh +grub-install --target=x86_64-efi --efi-directory=/boot --bootloader-id=GRUB +grub-mkconfig -o /boot/grub/grub.cfg ``` ## (Private) My Service Infrastructure Management @@ -23,14 +29,14 @@ All commands should be run at the project root path. Script from <https://docs.deno.com/runtime/getting_started/installation/> -```bash-session -$ curl -fsSL https://deno.land/install.sh | sh +```sh +curl -fsSL https://deno.land/install.sh | sh ``` ### Add Git Server User / Set Password -```bash-session -$ docker run -it --rm -v "./data/git/user-info:/user-info" httpd htpasswd /user-info [username] +```sh +docker run -it --rm -v "./data/git/user-info:/user-info" httpd htpasswd /user-info [username] ``` ### Certbot @@ -54,14 +60,14 @@ A complete command is `[prefix] [docker (based on challenge kind)] [command] [ch For example, **test** create/expand/shrink with standalone server: -```bash-session -# docker run -it --rm --name certbot \ -> -v "./data/certbot/certs:/etc/letsencrypt" -v "./data/certbot/data:/var/lib/letsencrypt"` \ -> -p "0.0.0.0:80:80" \ -> certonly \ -> --standalone \ -> -d crupest.life -d mail.crupest.life \ -> --test-cert --dry-run +```sh +docker run -it --rm --name certbot \ + -v "./data/certbot/certs:/etc/letsencrypt" -v "./data/certbot/data:/var/lib/letsencrypt"` \ + -p "0.0.0.0:80:80" \ + certonly \ + --standalone \ + -d crupest.life -d mail.crupest.life \ + --test-cert --dry-run ``` ## System Setup @@ -70,107 +76,57 @@ For example, **test** create/expand/shrink with standalone server: #### Setup SSL Certificates and Curl -```bash-session -# apt-get update -# apt-get install ca-certificates curl -# install -m 0755 -d /etc/apt/keyrings +```sh +apt-get update +apt-get install ca-certificates curl +install -m 0755 -d /etc/apt/keyrings ``` ### Docker Setup #### Uninstall Packages Provided by Stock Repo -```bash-session -# for pkg in docker.io docker-doc docker-compose \ -> podman-docker containerd runc; do -> apt-get remove $pkg; -> done +```bash +for pkg in docker.io docker-doc docker-compose \ + podman-docker containerd runc; do + apt-get remove $pkg; +done ``` #### Install Certs From Docker Remember to [setup ssl and curl](#setup-ssl-certificates-and-curl) first. -```bash-session -# curl -fsSL https://download.docker.com/linux/debian/gpg \ -> -o /etc/apt/keyrings/docker.asc -# chmod a+r /etc/apt/keyrings/docker.asc +```sh +curl -fsSL https://download.docker.com/linux/debian/gpg \ + -o /etc/apt/keyrings/docker.asc +chmod a+r /etc/apt/keyrings/docker.asc ``` #### Add Docker Repos -```bash-session -# echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] \ -> https://download.docker.com/linux/debian \ -> $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ -> tee /etc/apt/sources.list.d/docker.list > /dev/null +```bash +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] \ + https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null ``` #### Install Docker Packages -```bash-session -# apt-get update -# apt-get install docker-ce docker-ce-cli containerd.io \ -> docker-buildx-plugin docker-compose-plugin +```sh +apt-get update +apt-get install docker-ce docker-ce-cli containerd.io \ + docker-buildx-plugin docker-compose-plugin ``` #### Start And Enable Docker Remember to log out and log back to let user group change take effects. -```bash-session -# systemctl enable docker -# systemctl start docker -# groupadd -f docker -# usermod -aG docker $USER -``` - -### Font Config - -```xml -<?xml version="1.0"?> -<!DOCTYPE fontconfig SYSTEM "urn:fontconfig:fonts.dtd"> -<fontconfig> - - <alias> - <family>sans-serif</family> - <prefer> - <family>MiSans</family> - <family>Noto Color Emoji</family> - </prefer> - </alias> - - <alias> - <family>serif</family> - <prefer> - <family>MiSans</family> - <family>Noto Color Emoji</family> - </prefer> - </alias> - - <alias> - <family>Maple Mono</family> - <prefer> - <family>Maple Mono NF</family> - <family>Maple Mono</family> - </prefer> - </alias> - - <alias> - <family>Noto Sans</family> - <prefer> - <family>Noto Sans</family> - <family>Noto Sans CJK SC</family> - </prefer> - </alias> - - <alias> - <family>Noto Serif</family> - <prefer> - <family>Noto Serif</family> - <family>Noto Serif CJK SC</family> - </prefer> - </alias> - -</fontconfig> +```sh +systemctl enable docker +systemctl start docker +groupadd -f docker +usermod -aG docker $USER ``` diff --git a/www/content/notes/hurd.md b/www/content/notes/hurd.md index 67b8df9..aeb9b15 100644 --- a/www/content/notes/hurd.md +++ b/www/content/notes/hurd.md @@ -1,12 +1,12 @@ --- title: "Hurd" date: 2025-03-03T15:34:41+08:00 -lastmod: 2025-03-03T23:28:46+08:00 +lastmod: 2025-06-12T01:09:39+08:00 --- {{< mono >}} -[TODOS](/notes/hurd/todos) +goto: [Cheat Sheet](/notes/hurd/cheat-sheet) | [Todos](/notes/hurd/todos) {{< /mono >}} @@ -38,6 +38,18 @@ refs: {{< /mono >}} +## mailing lists / irc + +{{< mono >}} + +| name | address | +| --- | --- | +| hurd | <bug-hurd@gnu.org> | +| debian | <debian-hurd@lists.debian.org> | +| irc | librechat #hurd | + +{{< /mono >}} + ## *_MAX patch See [this](posts/c-func-ext.md) @@ -92,42 +104,3 @@ web cru: <https://crupest.life/git/hurd/web.git> upstream: <https://git.savannah.gnu.org/git/hurd/web.git> {{< /link-group >}} - -## cheatsheet - -Start qemu - -```sh -qemu-system-x86_64 -enable-kvm -m 4G \ - -net nic -net user,hostfwd=tcp::3222-:22 \ - -vga vmware -drive cache=writeback,file=[...] -``` - -Configure/Setup network - -```sh -settrans -fgap /servers/socket/2 /hurd/pfinet \ - -i /dev/eth0 -a 10.0.2.15 -g 10.0.2.2 -m 255.255.255.0 -fsysopts /servers/socket/2 /hurd/pfinet \ - -i /dev/eth0 -a 10.0.2.15 -g 10.0.2.2 -m 255.255.255.0 -fsysopts /server/socket/2 -a 10.0.2.15 -g 10.0.2.2 -m 255.255.255.0 -``` - -Setup apt - -```sh -apt-get --allow-unauthenticated --allow-insecure-repositories update -apt-get --allow-unauthenticated upgrade -``` - -## mailing lists / irc - -{{< mono >}} - -| name | address | -| --- | --- | -| hurd | <bug-hurd@gnu.org> | -| debian | <debian-hurd@lists.debian.org> | -| irc | librechat #hurd | - -{{< /mono >}} diff --git a/www/content/notes/hurd/cheat-sheet.md b/www/content/notes/hurd/cheat-sheet.md new file mode 100644 index 0000000..f48e943 --- /dev/null +++ b/www/content/notes/hurd/cheat-sheet.md @@ -0,0 +1,65 @@ +--- +title: "Hurd Cheat Sheet" +date: 2025-06-12T00:59:16+08:00 +lastmod: 2025-06-12T00:59:16+08:00 +--- + +## Mirrors + +The mirror has to be `debian-ports`, not `debian`, and many mirror sites do not +provide it. Following is aliyun mirror: + +```txt +/etc/apt/sources.list +--- +deb https://mirrors.aliyun.com/debian-ports/ unstable main +deb https://mirrors.aliyun.com/debian-ports/ unreleased main +deb-src https://mirrors.aliyun.com/debian/ unstable main +``` + +The hurd-amd64 deb-src seems to not work. + +## Use QEMU Virtual Machine + +For i386, use + +```sh +qemu-system-x86_64 -enable-kvm -m 4G \ + -net nic -net user,hostfwd=tcp::3222-:22 \ + -vga vmware -drive cache=writeback,file=[...] +``` + +For x86_64, use + +```sh +qemu-system-x86_64 -enable-kvm -m 8G -machine q35 \ + -net nic -net user,hostfwd=tcp::3223-:22 \ + -vga vmware -drive cache=writeback,file=[...] +``` + +GRUB in the image seems to use hard-coded path of `/dev/*` block file as the +root partition in the kernel command line rather than GUID, so if the hard disk +bus is changed in QEMU and the path is changed accordingly, the system can't +boot on. + +QEMU cli arguments `-machine q35` enables AHCI and SATA, and is **required for +official x86_64 image to boot**. As for i386, I haven't checked now. + +## Inside Hurd + +Configure/Setup network + +```sh +settrans -fgap /servers/socket/2 /hurd/pfinet \ + -i /dev/eth0 -a 10.0.2.15 -g 10.0.2.2 -m 255.255.255.0 +fsysopts /servers/socket/2 /hurd/pfinet \ + -i /dev/eth0 -a 10.0.2.15 -g 10.0.2.2 -m 255.255.255.0 +fsysopts /server/socket/2 -a 10.0.2.15 -g 10.0.2.2 -m 255.255.255.0 +``` + +Setup apt + +```sh +apt-get --allow-unauthenticated --allow-insecure-repositories update +apt-get --allow-unauthenticated upgrade +``` diff --git a/www/content/notes/hurd/todos.md b/www/content/notes/hurd/todos.md index cd93f01..8fe068b 100644 --- a/www/content/notes/hurd/todos.md +++ b/www/content/notes/hurd/todos.md @@ -58,5 +58,3 @@ gerrit: <https://chromium-review.googlesource.com/c/codecs/libgav1/+/6239812> {{< /link-group >}} {{< /todo >}} - - diff --git a/www/content/posts/c-func-ext.md b/www/content/posts/c-func-ext.md index 7106fad..1f5f822 100644 --- a/www/content/posts/c-func-ext.md +++ b/www/content/posts/c-func-ext.md @@ -8,6 +8,8 @@ tags: - posix --- +(I've given up on this, at least for linux pam.) + Recently, I’ve been working on porting some libraries to GNU/Hurd. Many (old) libraries use [`*_MAX` constants on POSIX system interfaces](https://pubs.opengroup.org/onlinepubs/9699919799.2008edition/nframe.html) diff --git a/www/layouts/index.html b/www/layouts/index.html index abf8a74..af3e11d 100644 --- a/www/layouts/index.html +++ b/www/layouts/index.html @@ -26,7 +26,7 @@ <hr /> <section> <p>Welcome to my home page! Nice to meet you here! 🥰</p> - <p>Feel free to contact me via my email address *<crupest@crupest.life>*, + <p>Feel free to contact me via my email address <a href="mailto:crupest@crupest.life">crupest@crupest.life</a>, or just create an issue in any of my <a rel="noopener noreferrer" href="https://github.com/crupest">GitHub</a> repos. I love talking with people a lot.</p> <div id="links" class="mono-link"> |