From 7b09e420ede2f9c92f5eaff28508ec10255414b7 Mon Sep 17 00:00:00 2001 From: Yuqian Yang Date: Fri, 13 Jun 2025 13:55:05 +0800 Subject: deno: move service manager to tools. --- deno/tools/deno.json | 1 + deno/tools/gen-geosite-rules.ts | 161 ----------------------------------- deno/tools/generate-geosite-rules.ts | 161 +++++++++++++++++++++++++++++++++++ deno/tools/manage-service.ts | 39 +++++++++ deno/tools/template.ts | 122 ++++++++++++++++++++++++++ 5 files changed, 323 insertions(+), 161 deletions(-) delete mode 100644 deno/tools/gen-geosite-rules.ts create mode 100644 deno/tools/generate-geosite-rules.ts create mode 100644 deno/tools/manage-service.ts create mode 100644 deno/tools/template.ts (limited to 'deno/tools') diff --git a/deno/tools/deno.json b/deno/tools/deno.json index 3597182..355046a 100644 --- a/deno/tools/deno.json +++ b/deno/tools/deno.json @@ -3,5 +3,6 @@ "tasks": { }, "imports": { + "mustache": "npm:mustache@^4.2.0" } } diff --git a/deno/tools/gen-geosite-rules.ts b/deno/tools/gen-geosite-rules.ts deleted file mode 100644 index c59d34f..0000000 --- a/deno/tools/gen-geosite-rules.ts +++ /dev/null @@ -1,161 +0,0 @@ -const PROXY_NAME = "node-select" -const ATTR = "cn" -const REPO_NAME = "domain-list-community"; -const URL = "https://github.com/v2fly/domain-list-community/archive/refs/heads/master.zip" -const SITES = [ - "github", - "google", - "youtube", - "twitter", - "facebook", - "discord", - "reddit", - "twitch", - "quora", - "telegram", - "imgur", - "stackexchange", - "onedrive", - "duckduckgo", - "wikimedia", - "gitbook", - "gitlab", - "creativecommons", - "archive", - "matrix", - "tor", - "python", - "ruby", - "rust", - "nodejs", - "npmjs", - "qt", - "docker", - "v2ray", - "homebrew", - "bootstrap", - "heroku", - "vercel", - "ieee", - "sci-hub", - "libgen", -] - -const prefixes = ["include", "domain", "keyword", "full", "regexp"] as const - -interface Rule { - kind: (typeof prefixes)[number]; - value: string; - attrs: string[]; -} - -type FileProvider = (name: string) => string; - -function extract(starts: string[], provider: FileProvider): Rule[] { -function parseLine(line: string): Rule { - let kind = prefixes.find((p) => line.startsWith(p + ":")); - if (kind != null) { - line = line.slice(line.indexOf(":") + 1); - } else { - kind = "domain"; - } - const segs = line.split("@"); - return { - kind, - value: segs[0].trim(), - attrs: [...segs.slice(1)].map((s) => s.trim()), - }; -} - - function parse(text: string): Rule[] { - return text - .replaceAll("\c\n", "\n") - .split("\n") - .map((l) => l.trim()) - .filter((l) => l.length !== 0 && !l.startsWith("#")) - .map((l) => parseLine(l)); - } - - const visited = [] as string[] - const rules = [] as Rule[] - - function add(name :string) { - const text = provider(name); - for (const rule of parse(text)) { - if (rule.kind === "include") { - if (visited.includes(rule.value)) { - console.warn(`circular refs found: ${name} includes ${rule.value}.`); - continue; - } else { - visited.push(rule.value); - add(rule.value); - } - } else { - rules.push(rule); - } - } - } - - for (const start of starts) { - add(start); - } - - return rules -} - -function toNewFormat(rules: Rule[], attr: string): [string, string] { - function toLine(rule: Rule) { - const prefixMap = { - "domain": "DOMAIN-SUFFIX", - "full": "DOMAIN", - "keyword": "DOMAIN-KEYWORD", - "regexp": "DOMAIN-REGEX", - } as const; - if (rule.kind === "include") { - throw new Error("Include rule not parsed.") - } - return `${prefixMap[rule.kind]},${rule.value}` - } - - function toLines(rules: Rule[]) { - return rules.map(r => toLine(r)).join("\n") - } - - const has: Rule[] = []; - const notHas: Rule[] = []; - rules.forEach((r) => (r.attrs.includes(attr) ? has.push(r) : notHas.push(r))); - - return [toLines(has), toLines(notHas)]; -} - - -if (import.meta.main) { - const tmpDir = Deno.makeTempDirSync({ prefix: "geosite-rules-" }); - console.log("Work dir is ", tmpDir); - const zipFilePath = tmpDir + "/repo.zip"; - const res = await fetch(URL); - if (!res.ok) { - throw new Error("Failed to download repo."); - } - Deno.writeFileSync(zipFilePath, await res.bytes()); - const unzip = new Deno.Command("unzip", { - args: ["-q", zipFilePath], - cwd: tmpDir, - }); - if (!(await unzip.spawn().status).success) { - throw new Error("Failed to unzip"); - } - - const dataDir = tmpDir + "/" + REPO_NAME + "-master/data"; - const provider = (name: string) => - Deno.readTextFileSync(dataDir + "/" + name); - - const rules = extract(SITES, provider) - const [has, notHas] = toNewFormat(rules, ATTR) - const hasFile = tmpDir + "/has-rule" - const notHasFile = tmpDir + "/not-has-rule" - console.log("Write result to: " + hasFile + " , " + notHasFile) - Deno.writeTextFileSync(hasFile, has) - Deno.writeTextFileSync(notHasFile, notHas) -} - diff --git a/deno/tools/generate-geosite-rules.ts b/deno/tools/generate-geosite-rules.ts new file mode 100644 index 0000000..c59d34f --- /dev/null +++ b/deno/tools/generate-geosite-rules.ts @@ -0,0 +1,161 @@ +const PROXY_NAME = "node-select" +const ATTR = "cn" +const REPO_NAME = "domain-list-community"; +const URL = "https://github.com/v2fly/domain-list-community/archive/refs/heads/master.zip" +const SITES = [ + "github", + "google", + "youtube", + "twitter", + "facebook", + "discord", + "reddit", + "twitch", + "quora", + "telegram", + "imgur", + "stackexchange", + "onedrive", + "duckduckgo", + "wikimedia", + "gitbook", + "gitlab", + "creativecommons", + "archive", + "matrix", + "tor", + "python", + "ruby", + "rust", + "nodejs", + "npmjs", + "qt", + "docker", + "v2ray", + "homebrew", + "bootstrap", + "heroku", + "vercel", + "ieee", + "sci-hub", + "libgen", +] + +const prefixes = ["include", "domain", "keyword", "full", "regexp"] as const + +interface Rule { + kind: (typeof prefixes)[number]; + value: string; + attrs: string[]; +} + +type FileProvider = (name: string) => string; + +function extract(starts: string[], provider: FileProvider): Rule[] { +function parseLine(line: string): Rule { + let kind = prefixes.find((p) => line.startsWith(p + ":")); + if (kind != null) { + line = line.slice(line.indexOf(":") + 1); + } else { + kind = "domain"; + } + const segs = line.split("@"); + return { + kind, + value: segs[0].trim(), + attrs: [...segs.slice(1)].map((s) => s.trim()), + }; +} + + function parse(text: string): Rule[] { + return text + .replaceAll("\c\n", "\n") + .split("\n") + .map((l) => l.trim()) + .filter((l) => l.length !== 0 && !l.startsWith("#")) + .map((l) => parseLine(l)); + } + + const visited = [] as string[] + const rules = [] as Rule[] + + function add(name :string) { + const text = provider(name); + for (const rule of parse(text)) { + if (rule.kind === "include") { + if (visited.includes(rule.value)) { + console.warn(`circular refs found: ${name} includes ${rule.value}.`); + continue; + } else { + visited.push(rule.value); + add(rule.value); + } + } else { + rules.push(rule); + } + } + } + + for (const start of starts) { + add(start); + } + + return rules +} + +function toNewFormat(rules: Rule[], attr: string): [string, string] { + function toLine(rule: Rule) { + const prefixMap = { + "domain": "DOMAIN-SUFFIX", + "full": "DOMAIN", + "keyword": "DOMAIN-KEYWORD", + "regexp": "DOMAIN-REGEX", + } as const; + if (rule.kind === "include") { + throw new Error("Include rule not parsed.") + } + return `${prefixMap[rule.kind]},${rule.value}` + } + + function toLines(rules: Rule[]) { + return rules.map(r => toLine(r)).join("\n") + } + + const has: Rule[] = []; + const notHas: Rule[] = []; + rules.forEach((r) => (r.attrs.includes(attr) ? has.push(r) : notHas.push(r))); + + return [toLines(has), toLines(notHas)]; +} + + +if (import.meta.main) { + const tmpDir = Deno.makeTempDirSync({ prefix: "geosite-rules-" }); + console.log("Work dir is ", tmpDir); + const zipFilePath = tmpDir + "/repo.zip"; + const res = await fetch(URL); + if (!res.ok) { + throw new Error("Failed to download repo."); + } + Deno.writeFileSync(zipFilePath, await res.bytes()); + const unzip = new Deno.Command("unzip", { + args: ["-q", zipFilePath], + cwd: tmpDir, + }); + if (!(await unzip.spawn().status).success) { + throw new Error("Failed to unzip"); + } + + const dataDir = tmpDir + "/" + REPO_NAME + "-master/data"; + const provider = (name: string) => + Deno.readTextFileSync(dataDir + "/" + name); + + const rules = extract(SITES, provider) + const [has, notHas] = toNewFormat(rules, ATTR) + const hasFile = tmpDir + "/has-rule" + const notHasFile = tmpDir + "/not-has-rule" + console.log("Write result to: " + hasFile + " , " + notHasFile) + Deno.writeTextFileSync(hasFile, has) + Deno.writeTextFileSync(notHasFile, notHas) +} + diff --git a/deno/tools/manage-service.ts b/deno/tools/manage-service.ts new file mode 100644 index 0000000..45e09ec --- /dev/null +++ b/deno/tools/manage-service.ts @@ -0,0 +1,39 @@ +import { parseArgs } from "@std/cli"; +import { TemplateDir } from "./template.ts"; +import { join } from "@std/path"; + +if (import.meta.main) { + const args = parseArgs(Deno.args, { + string: ["project-dir"], + boolean: ["no-dry-run"], + }); + + if (args._.length === 0) { + throw new Error("You must specify a command."); + } + + const projectDir = args["project-dir"]; + if (projectDir == null) { + throw new Error("You must specify project-dir."); + } + + const command = String(args._[0]); + + switch (command) { + case "gen-tmpl": + new TemplateDir( + join(projectDir, "services/templates"), + ).generateWithVariableFiles( + [ + join(projectDir, "data/config"), + join(projectDir, "services/config.template"), + ], + args["no-dry-run"] === true + ? join(projectDir, "services/generated") + : undefined, + ); + break; + default: + throw new Error(command + " is not a valid command."); + } +} diff --git a/deno/tools/template.ts b/deno/tools/template.ts new file mode 100644 index 0000000..0b043a1 --- /dev/null +++ b/deno/tools/template.ts @@ -0,0 +1,122 @@ +import { dirname, join, relative } from "@std/path"; +import { copySync, existsSync, walkSync } from "@std/fs"; +import { parse } from "@std/dotenv"; +import { distinct } from "@std/collections"; +// @ts-types="npm:@types/mustache" +import Mustache from "mustache"; + +Mustache.tags = ["@@", "@@"]; +Mustache.escape = (value) => String(value); + +function getVariableKeys(original: string): string[] { + return distinct( + Mustache.parse(original) + .filter(function (v) { + return v[0] === "name"; + }) + .map(function (v) { + return v[1]; + }), + ); +} + +export function loadVariables(files: string[]): Record { + const vars: Record = {}; + for (const file of files) { + const text = Deno.readTextFileSync(file); + for (const [key, valueText] of Object.entries(parse(text))) { + getVariableKeys(valueText).forEach((name) => { + if (!(name in vars)) { + throw new Error( + `Variable ${name} is not defined yet, perhaps due to typos or wrong order.`, + ); + } + }); + vars[key] = Mustache.render(valueText, vars); + } + } + return vars; +} + +const TEMPLATE_FILE_EXT = ".template"; + +export class TemplateDir { + templates: { path: string; ext: string; text: string; vars: string[] }[] = []; + plains: { path: string }[] = []; + + constructor(public dir: string) { + console.log("Scanning template dir:"); + Array.from( + walkSync(dir, { includeDirs: false, followSymlinks: true }), + ).forEach(({ path }) => { + path = relative(this.dir, path); + if (path.endsWith(TEMPLATE_FILE_EXT)) { + console.log(` (template) ${path}`); + const text = Deno.readTextFileSync(join(dir, path)); + this.templates.push({ + path, + ext: TEMPLATE_FILE_EXT, + text, + vars: getVariableKeys(text), + }); + } else { + console.log(` (plain) ${path}`); + this.plains.push({ path }); + } + }); + console.log("Done scanning template dir."); + } + + allNeededVars() { + return distinct(this.templates.flatMap((t) => t.vars)); + } + + generate(vars: Record, generatedDir?: string) { + console.log( + `Generating, template dir: ${this.dir}, generated dir: ${generatedDir ?? "[dry-run]"}:`, + ); + + const undefinedVars = this.allNeededVars().filter((v) => !(v in vars)); + if (undefinedVars.length !== 0) { + throw new Error( + `Needed variables are not defined: ${undefinedVars.join(", ")}`, + ); + } + + if (generatedDir != null) { + if (existsSync(generatedDir)) { + console.log(` delete old generated dir ${generatedDir}`); + Deno.removeSync(generatedDir, { recursive: true }); + } + + for (const file of this.plains) { + const [source, destination] = [ + join(this.dir, file.path), + join(generatedDir, file.path), + ]; + console.log(` copy ${source} to ${destination} ...`); + Deno.mkdirSync(dirname(destination), { recursive: true }); + copySync(source, destination); + } + for (const file of this.templates) { + const [source, destination] = [ + join(this.dir, file.path), + join(generatedDir, file.path.slice(0, -file.ext.length)), + ]; + console.log(` generate ${source} to ${destination} ...`); + const rendered = Mustache.render(file.text, vars); + Deno.mkdirSync(dirname(destination), { recursive: true }); + Deno.writeTextFileSync(destination, rendered); + } + } + console.log(`Done generating.`); + } + + generateWithVariableFiles(varFiles: string[], generatedDir?: string) { + console.log("Scanning defined vars:"); + const vars = loadVariables(varFiles); + Object.keys(vars).forEach((name) => console.log(` ${name}`)); + console.log("Done scanning defined vars."); + this.generate(vars, generatedDir); + } +} -- cgit v1.2.3