From 5fe53ec36e9298fd40a36877a9a5ceb791814f50 Mon Sep 17 00:00:00 2001 From: crupest Date: Sat, 19 Nov 2022 13:00:13 +0800 Subject: No caddy, only nginx and certbot. --- .gitignore | 3 +- requirements.txt | 2 + template/README.md | 2 +- template/docker-compose.yaml.template | 5 + template/nginx/https-redirect.conf | 12 + template/nginx/reverse-proxy.conf.template | 23 ++ template/nginx/root.conf.template | 10 + template/nginx/server.json | 25 ++ template/nginx/server.schema.json | 80 ++++++ template/nginx/server.ts | 29 +++ template/nginx/ssl.conf | 14 ++ template/nginx/static-file.conf.template | 10 + tool/aio.py | 381 +++++++++++++++++++++++++++++ tool/modules/configfile.py | 45 ++++ tool/modules/nginx.py | 72 ++++++ tool/modules/path.py | 13 + tool/modules/template.py | 32 +++ tool/setup.py | 377 ---------------------------- 18 files changed, 756 insertions(+), 379 deletions(-) create mode 100644 requirements.txt create mode 100644 template/nginx/https-redirect.conf create mode 100644 template/nginx/reverse-proxy.conf.template create mode 100644 template/nginx/root.conf.template create mode 100644 template/nginx/server.json create mode 100644 template/nginx/server.schema.json create mode 100644 template/nginx/server.ts create mode 100644 template/nginx/ssl.conf create mode 100644 template/nginx/static-file.conf.template create mode 100755 tool/aio.py create mode 100644 tool/modules/configfile.py create mode 100755 tool/modules/nginx.py create mode 100644 tool/modules/path.py create mode 100644 tool/modules/template.py delete mode 100755 tool/setup.py diff --git a/.gitignore b/.gitignore index c24b8a5..95d34cf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -Caddyfile docker-compose.yaml mailserver.env data +nginx-config +__pycache__ \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..11c8a40 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +rich +jsonschema diff --git a/template/README.md b/template/README.md index 5228d6b..729e596 100644 --- a/template/README.md +++ b/template/README.md @@ -1,4 +1,4 @@ -This directory contains the template files used to generate the final config files. They should not be used directly usually. Run `../tool/setup.py` to perform any generation and necessary setup. +This directory contains the template files used to generate the final config files. They should not be used directly usually. Run `../tool/aio.py` to perform any generation and necessary setup. The template format is quite simple: they are just files containing `$VAR`s or `${VAR}`s. All variable names begin with `CRUPEST_` to avoid name conflicts. diff --git a/template/docker-compose.yaml.template b/template/docker-compose.yaml.template index ef6115f..35027c9 100644 --- a/template/docker-compose.yaml.template +++ b/template/docker-compose.yaml.template @@ -58,6 +58,11 @@ services: - "80:80" - "443:443" volumes: + - "./nginx-config:/etc/nginx/conf.d:ro" + - "./data/certbot/certs:/etc/letsencrypt:ro" + networks: + - external + - internal mailserver: image: docker.io/mailserver/docker-mailserver:latest diff --git a/template/nginx/https-redirect.conf b/template/nginx/https-redirect.conf new file mode 100644 index 0000000..6301836 --- /dev/null +++ b/template/nginx/https-redirect.conf @@ -0,0 +1,12 @@ +server { + listen 80 default_server; + listen [::]:80 default_server; + + location / { + return 301 https://$host$request_uri; + } + + location /.well-known/acme-challenge { + root /srv/acme; + } +} diff --git a/template/nginx/reverse-proxy.conf.template b/template/nginx/reverse-proxy.conf.template new file mode 100644 index 0000000..e98c066 --- /dev/null +++ b/template/nginx/reverse-proxy.conf.template @@ -0,0 +1,23 @@ +upstream ${CRUPEST_NGINX_UPSTREAM_NAME} { + server ${CRUPEST_NGINX_UPSTREAM_SERVER}; +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name ${CRUPEST_NGINX_SUBDOMAIN}.${CRUPEST_DOMAIN}; + + ssl_certificate /etc/letsencrypt/live/${CRUPEST_NGINX_SUBDOMAIN}.${CRUPEST_DOMAIN}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/${CRUPEST_NGINX_SUBDOMAIN}.${CRUPEST_DOMAIN}/privkey.pem; + + location / { + proxy_pass http://${CRUPEST_NGINX_UPSTREAM_NAME}; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $http_connection; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + } +} diff --git a/template/nginx/root.conf.template b/template/nginx/root.conf.template new file mode 100644 index 0000000..8af8fff --- /dev/null +++ b/template/nginx/root.conf.template @@ -0,0 +1,10 @@ +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name ${CRUPEST_DOMAIN}; + + ssl_certificate /etc/letsencrypt/live/${CRUPEST_DOMAIN}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/${CRUPEST_DOMAIN}/privkey.pem; + + root /srv/www; +} diff --git a/template/nginx/server.json b/template/nginx/server.json new file mode 100644 index 0000000..cad0cb3 --- /dev/null +++ b/template/nginx/server.json @@ -0,0 +1,25 @@ +{ + "$schema": "./server.schema.json", + "sites": [ + { + "type": "reverse-proxy", + "subdomain": "code", + "upstream": { + "name": "code-server", + "server": "code-server:8080" + } + }, + { + "type": "reverse-proxy", + "subdomain": "halo", + "upstream": { + "name": "halo", + "server": "halo:8090" + } + }, + { + "type": "cert-only", + "subdomain": "mail" + } + ] +} \ No newline at end of file diff --git a/template/nginx/server.schema.json b/template/nginx/server.schema.json new file mode 100644 index 0000000..536fead --- /dev/null +++ b/template/nginx/server.schema.json @@ -0,0 +1,80 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "CertOnlySite": { + "properties": { + "subdomain": { + "type": "string" + }, + "type": { + "enum": [ + "cert-only" + ], + "type": "string" + } + }, + "type": "object" + }, + "ReverseProxySite": { + "properties": { + "subdomain": { + "type": "string" + }, + "type": { + "enum": [ + "reverse-proxy" + ], + "type": "string" + }, + "upstream": { + "properties": { + "name": { + "type": "string" + }, + "server": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "StaticFileSite": { + "properties": { + "root": { + "type": "string" + }, + "subdomain": { + "type": "string" + }, + "type": { + "enum": [ + "static-file" + ], + "type": "string" + } + }, + "type": "object" + } + }, + "properties": { + "sites": { + "items": { + "anyOf": [ + { + "$ref": "#/definitions/ReverseProxySite" + }, + { + "$ref": "#/definitions/StaticFileSite" + }, + { + "$ref": "#/definitions/CertOnlySite" + } + ] + }, + "type": "array" + } + }, + "type": "object" +} \ No newline at end of file diff --git a/template/nginx/server.ts b/template/nginx/server.ts new file mode 100644 index 0000000..6a5d24d --- /dev/null +++ b/template/nginx/server.ts @@ -0,0 +1,29 @@ +// Used to generate json schema. + +export interface ReverseProxySite { + type: "reverse-proxy"; + subdomain: string; + upstream: { + name: string; + server: string; + }; +} + +export interface StaticFileSite { + type: "static-file"; + subdomain: string; + root: string; +} + +export interface CertOnlySite { + type: "cert-only"; + subdomain: string; +} + +export type Site = ReverseProxySite | StaticFileSite | CertOnlySite; + +export type Sites = Site[]; + +export interface Server { + sites: Sites; +} diff --git a/template/nginx/ssl.conf b/template/nginx/ssl.conf new file mode 100644 index 0000000..f2aadba --- /dev/null +++ b/template/nginx/ssl.conf @@ -0,0 +1,14 @@ +# This file contains important security parameters. If you modify this file +# manually, Certbot will be unable to automatically provide future security +# updates. Instead, Certbot will print and log an error message with a path to +# the up-to-date file that you will need to refer to when manually updating +# this file. Contents are based on https://ssl-config.mozilla.org + +ssl_session_cache shared:le_nginx_SSL:10m; +ssl_session_timeout 1440m; +ssl_session_tickets off; + +ssl_protocols TLSv1.2 TLSv1.3; +ssl_prefer_server_ciphers off; + +ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384"; diff --git a/template/nginx/static-file.conf.template b/template/nginx/static-file.conf.template new file mode 100644 index 0000000..01054cf --- /dev/null +++ b/template/nginx/static-file.conf.template @@ -0,0 +1,10 @@ +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name ${CRUPEST_NGINX_SUBDOMAIN}.${CRUPEST_DOMAIN}; + + ssl_certificate /etc/letsencrypt/live/${CRUPEST_NGINX_SUBDOMAIN}.${CRUPEST_DOMAIN}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/${CRUPEST_NGINX_SUBDOMAIN}.${CRUPEST_DOMAIN}/privkey.pem; + + root ${CRUPEST_NGINX_ROOT}; +} diff --git a/tool/aio.py b/tool/aio.py new file mode 100755 index 0000000..23540bb --- /dev/null +++ b/tool/aio.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 + +import os +import os.path +import pwd +import grp +import sys +import argparse +import typing +import shutil +import urllib.request +from rich.console import Console +from rich.prompt import Prompt, Confirm +from modules.path import * +from modules.template import Template +from modules.nginx import * +from modules.configfile import * + +console = Console() + + +def print_order(number: int, total: int, *, console=console) -> None: + console.print(f"\[{number}/{total}]", end=" ", style="green") + + +parser = argparse.ArgumentParser( + description="Crupest server all-in-one setup script. Have fun play with it!") +subparsers = parser.add_subparsers(dest="action") + +setup_parser = subparsers.add_parser( + "setup", help="Do everything necessary to setup the server.") + +download_tools_parser = subparsers.add_parser( + "download-tools", help="Download some extra tools to manage the server.") + +domain_parser = subparsers.add_parser( + "domain", help="Misc things about domains.") +domain_subparsers = domain_parser.add_subparsers(dest="domain_action") + +domain_list_parser = domain_subparsers.add_parser( + "list", help="List all domains.") + +domain_nginx_parser = domain_subparsers.add_parser( + "nginx", help="Generate nginx config for a domain.") + +domain_certbot_parser = domain_subparsers.add_parser( + "certbot", help="Get some common certbot commands.") + +domain_certbot_parser.add_argument( + "-t", "--test", action="store_true", help="Make the commands for test use.") + +clear_parser = subparsers .add_parser( + "clear", help="Delete existing data so you can make a fresh start.") +clear_parser.add_argument("-D", "--include-data-dir", action="store_true", + default=False, help="Also delete the data directory.") + +args = parser.parse_args() + +console.print("Nice to see you! :waving_hand:", style="cyan") + + +def check_domain_is_defined() -> str: + try: + return get_domain() + except ValueError as e: + console.print( + "We are not able to get the domain. You may want to first run setup command.", style="red") + console.print_exception(e) + exit(1) + + +def download_tools(): + SCRIPTS = [("docker-mailserver setup script", "docker-mailserver-setup.sh", + "https://raw.githubusercontent.com/docker-mailserver/docker-mailserver/master/setup.sh")] + for index, script in enumerate(SCRIPTS): + number = index + 1 + total = len(SCRIPTS) + print_order(number, total) + name, filename, url = script + path = os.path.join(tool_dir, filename) + skip = False + if os.path.exists(path): + overwrite = Confirm.ask( + f"[cyan]{name}[/] already exists, download and overwrite?", default=False) + if not overwrite: + skip = True + else: + download = Confirm.ask( + f"Download [cyan]{name}[/] to [magenta]{path}[/]?", default=True) + if not download: + skip = True + if not skip: + console.print(f"Downloading {name}...") + urllib.request.urlretrieve(url, path) + os.chmod(path, 0o755) + console.print(f"Downloaded {name} to {path}.", style="green") + else: + console.print(f"Skipped {name}.", style="yellow") + + +def generate_nginx_config(domain: str) -> None: + if not os.path.exists(nginx_config_dir): + os.mkdir(nginx_config_dir) + console.print( + f"Nginx config directory created at [magenta]{nginx_config_dir}[/]", style="green") + nginx_config_gen(domain, dest=nginx_config_dir) + console.print("Nginx config generated.", style="green") + + +if args.action == 'domain': + domain = check_domain_is_defined() + match args.domain_action: + case 'list': + domains = list_domains(domain) + for domain in domains: + console.print(domain) + case 'certbot': + console.print( + "Here is some commands you can use to do certbot related work.") + is_test = args.test + if is_test: + console.print( + "Note you specified --test, so the commands are for test use.", style="yellow") + console.print( + f"To create certs for init:\n[code]{certbot_command_gen(domain, 'create', test=is_test)}[/]") + console.print( + f"To renew certs previously created:\n[code]{certbot_command_gen(domain, 'renew', test=is_test)}[/]") + case 'nginx': + generate_nginx_config(domain) + exit(0) + + +if args.action == 'download-tools': + download_tools() + exit(0) + +print("First let's check all the templates...") + +# get all filenames ending with .template +template_name_list = [os.path.basename(f)[:-len('.template')] for f in os.listdir( + template_dir) if f.endswith(".template")] + +# if action is 'clean' +if args.action == "clear": + # check root if we have to delete data dir + if args.include_data_dir and os.path.exists(data_dir) and os.geteuid() != 0: + console.print("You need to be root to delete data dir.", style="red") + sys.exit(1) + + to_delete = Confirm.ask( + "[yellow]Are you sure you want to delete everything? all your data will be lost![/]", default=False) + if to_delete: + files_to_delete = [] + for template_name in template_name_list: + f = os.path.join(project_dir, template_name) + if os.path.exists(f): + files_to_delete.append(f) + + delete_data_dir = args.include_data_dir and os.path.exists(data_dir) + + if len(files_to_delete) == 0: + console.print("Nothing to delete. We are safe!", style="green") + exit(0) + + console.print("Here are the files to delete:") + for f in files_to_delete: + console.print(f, style="magenta") + if delete_data_dir: + console.print(data_dir + " (data dir)", style="magenta") + + to_delete = Confirm.ask( + "[yellow]Are you sure you want to delete them?[/]", default=False) + if to_delete: + for f in files_to_delete: + os.remove(f) + if delete_data_dir: + # recursively delete data dir + shutil.rmtree(data_dir) + console.print( + "Your workspace is clean now! However config file is still there! See you!", style="green") + exit(0) + +console.print( + f"I have found following template files in [magenta]{template_dir}[/]:", style="green") +for filename in template_name_list: + console.print(f"- [magenta]{filename}.template[/]") + + +class ConfigVar: + def __init__(self, name: str, description: str, default_value_generator: typing.Callable[[], str] | str): + """Create a config var. + + Args: + name (str): The name of the config var. + description (str): The description of the config var. + default_value_generator (typing.Callable([], str) | str): The default value generator of the config var. If it is a string, it will be used as the input prompt and let user input the value. + """ + self.name = name + self.description = description + self.default_value_generator = default_value_generator + + def get_default_value(self): + if isinstance(self.default_value_generator, str): + return Prompt.ask(self.default_value_generator, console=console) + else: + return self.default_value_generator() + + +config_var_list: list[ConfigVar] = [ + ConfigVar("CRUPEST_DOMAIN", "domain name", + "Please input your domain name:"), + # ConfigVar("CRUPEST_EMAIL", "admin email address", + # "Please input your email address:"), + ConfigVar("CRUPEST_USER", "your system account username", + lambda: pwd.getpwuid(os.getuid()).pw_name), + ConfigVar("CRUPEST_GROUP", "your system account group name", + lambda: grp.getgrgid(os.getgid()).gr_name), + ConfigVar("CRUPEST_UID", "your system account uid", + lambda: str(os.getuid())), + ConfigVar("CRUPEST_GID", "your system account gid", + lambda: str(os.getgid())), + ConfigVar("CRUPEST_HALO_DB_PASSWORD", + "password for halo h2 database, once used never change it", lambda: os.urandom(8).hex()), + ConfigVar("CRUPEST_IN_CHINA", + "set to true if you are in China, some network optimization will be applied", lambda: "false") +] + +config_var_name_set = set([config_var.name for config_var in config_var_list]) + +template_list: list[Template] = [] +config_var_name_set_in_template = set() +for template_path in os.listdir(template_dir): + if not template_path.endswith(".template"): + continue + template = Template(os.path.join(template_dir, template_path)) + template_list.append(template) + config_var_name_set_in_template.update(template.var_set) + +console.print( + "I have found following variables needed in templates:", style="green") +for key in config_var_name_set_in_template: + console.print(key, end=" ", style="magenta") +console.print("") + +# check vars +if not config_var_name_set_in_template == config_var_name_set: + console.print( + "The variables needed in templates are not same to the explicitly declared ones! There must be something wrong.", style="red") + console.print("The explicitly declared ones are:") + for key in config_var_name_set: + console.print(key, end=" ", style="magenta") + console.print( + "\nTry to check template files and edit the var list at the head of this script. Aborted! See you next time!") + exit(1) + + +console.print("Now let's check if they are already generated...") + +conflict = False + +# check if there exists any generated files +for filename in template_name_list: + if os.path.exists(os.path.join(project_dir, filename)): + console.print(f"Found [magenta]{filename}[/]") + conflict = True + +if conflict: + to_overwrite = Confirm.ask( + "It seems there are some files already generated. Do you want to overwrite them?", console=console, default=False) + if not to_overwrite: + console.print( + "Great! Check the existing files and see you next time!", style="green") + exit() +else: + print("No conflict found. Let's go on!\n") + +console.print("Check for existing config file...") + + +# check if there exists a config file +if not config_file_exist: + config = {} + console.print( + "No existing config file found. Don't worry. Let's create one!", style="green") + for config_var in config_var_list: + config[config_var.name] = config_var.get_default_value() + config_content = config_to_str(config) + # create data dir if not exist + if not os.path.exists(data_dir): + os.mkdir(data_dir) + # write config file + with open(config_file_path, "w") as f: + f.write(config_content) + console.print( + f"Everything else is auto generated. The config file is written into [magenta]{config_file_path}[/]. You had better keep it well. And here is the content:", style="green") + print_config(console, config) + is_ok = Confirm.ask( + "If you think it's not ok, you can stop here and edit it. Or let's go on?", console=console, default=True) + if not is_ok: + console.print( + "Great! Check the config file and see you next time!", style="green") + exit() +else: + console.print( + "Looks like you have already had a config file. Let's check the content:", style="green") + with open(config_file_path, "r") as f: + content = f.read() + config = parse_config(content) + print_config(console, config) + missed_config_vars = [] + for config_var in config_var_list: + if config_var.name not in config: + missed_config_vars.append(config_var) + + if len(missed_config_vars) > 0: + console.print( + "Oops! It seems you have missed some keys in your config file. Let's add them!", style="green") + for config_var in missed_config_vars: + config[config_var.name] = config_var.get_default_value() + content = config_to_str(config) + with open(config_file_path, "w") as f: + f.write(content) + console.print( + f"Here is the new config, it has been written out to [magenta]{config_file_path}[/]:") + print_config(console, config) + good_enough = Confirm.ask("Is it good enough?", + console=console, default=True) + if not good_enough: + console.print( + "Great! Check the config file and see you next time!", style="green") + exit() + +console.print( + "Finally, everything is ready. Let's generate the files:", style="green") + +# generate files +for index, template in enumerate(template_list): + number = index + 1 + total = len(template_list) + print_order(number, total) + console.print( + f"Generating [magenta]{template.template_name}[/]...") + content = template.generate(config) + with open(os.path.join(project_dir, template.template_name), "w") as f: + f.write(content) + +# generate nginx config +if not os.path.exists(nginx_config_dir): + to_gen_nginx_conf = Confirm.ask("It seems you haven't generate nginx config. Do you want to generate it?", + default=True, console=console) +else: + to_gen_nginx_conf = Confirm.ask("It seems you have already generated nginx config. Do you want to overwrite it?", + default=False, console=console) +if to_gen_nginx_conf: + domain = config["CRUPEST_DOMAIN"] + generate_nginx_config(domain) + + +if not os.path.exists(os.path.join(data_dir, "code-server")): + os.mkdir(os.path.join(data_dir, "code-server")) + console.print( + "I also create data dir for code-server. Because letting docker create it would result in permission problem.", style="green") +else: + code_server_stat = os.stat(os.path.join(data_dir, "code-server")) + if code_server_stat.st_uid == 0 or code_server_stat.st_gid == 0: + console.print( + "WARNING: The owner of data dir for code-server is root. This may cause permission problem. You had better change it.", style="yellow") + to_fix = Confirm.ask( + "Do you want me to help you fix it?", console=console, default=True) + if to_fix: + os.system( + f"sudo chown -R {os.getuid()}:{os.getgid()} {os.path.join(data_dir, 'code-server')}") + +console.print(":beers: All done!", style="green") +to_download_tools = Confirm.ask( + "By the way, would you like to download some scripts to do some extra setup like creating email user?", console=console, default=True) +if not to_download_tools: + console.print("Great! See you next time!", style="green") + exit() + +download_tools() diff --git a/tool/modules/configfile.py b/tool/modules/configfile.py new file mode 100644 index 0000000..49f2200 --- /dev/null +++ b/tool/modules/configfile.py @@ -0,0 +1,45 @@ +import os.path +from .path import config_file_path + +config_file_exist = os.path.isfile(config_file_path) + + +def parse_config(str: str) -> dict: + config = {} + for line_number, line in enumerate(str.splitlines()): + # check if it's a comment + if line.startswith("#"): + continue + # check if there is a '=' + if line.find("=") == -1: + raise ValueError( + f"Invalid config string. Please check line {line_number + 1}. There is even no '='!") + # split at first '=' + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + config[key] = value + return config + + +def get_domain() -> str: + if not config_file_exist: + raise ValueError("Config file not found!") + with open(config_file_path) as f: + config = parse_config(f.read()) + if "CRUPEST_DOMAIN" not in config: + raise ValueError("Domain not found in config file!") + return config["CRUPEST_DOMAIN"] + + +def config_to_str(config: dict) -> str: + return "\n".join([f"{key}={value}" for key, value in config.items()]) + + +def print_config(console, config: dict) -> None: + for key, value in config.items(): + console.print(f"[magenta]{key}[/] = [cyan]{value}") + + +__all__ = ["config_file_exist", "parse_config", + "get_domain", "config_to_str", "print_config"] diff --git a/tool/modules/nginx.py b/tool/modules/nginx.py new file mode 100755 index 0000000..6cb918c --- /dev/null +++ b/tool/modules/nginx.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +from .template import Template +from .path import nginx_template_dir +import json +import jsonschema +import os +import os.path +import shutil + +with open(os.path.join(nginx_template_dir, 'server.json')) as f: + server = json.load(f) + +with open(os.path.join(nginx_template_dir, 'server.schema.json')) as f: + schema = json.load(f) + +jsonschema.validate(server, schema) + +root_template = Template(os.path.join( + nginx_template_dir, 'root.conf.template')) +static_file_template = Template(os.path.join( + nginx_template_dir, 'static-file.conf.template')) +reverse_proxy_template = Template(os.path.join( + nginx_template_dir, 'reverse-proxy.conf.template')) + + +def nginx_config_gen(domain: str, dest: str) -> None: + if not os.path.isdir(dest): + raise ValueError('dest must be a directory') + # copy ssl.conf and https-redirect.conf which need no variable substitution + for filename in ['ssl.conf', 'https-redirect.conf']: + src = os.path.join(nginx_template_dir, filename) + dst = os.path.join(dest, filename) + shutil.copyfile(src, dst) + config = {"CRUPEST_DOMAIN": domain} + # generate root.conf + with open(os.path.join(dest, f'{domain}.conf'), 'w') as f: + f.write(root_template.generate(config)) + # generate nginx config for each site + sites: list = server["sites"] + for site in sites: + if site["type"] not in ['static-file', 'reverse-proxy']: + continue + subdomain = site["subdomain"] + local_config = config.copy() + local_config['CRUPEST_NGINX_SUBDOMAIN'] = subdomain + match site["type"]: + case 'static-file': + template = static_file_template + local_config['CRUPEST_NGINX_ROOT'] = site["root"] + case 'reverse-proxy': + template = reverse_proxy_template + local_config['CRUPEST_NGINX_UPSTREAM_NAME'] = site["upstream"]["name"] + local_config['CRUPEST_NGINX_UPSTREAM_SERVER'] = site["upstream"]["server"] + with open(os.path.join(dest, f'{subdomain}.{domain}.conf'), 'w') as f: + f.write(template.generate(local_config)) + + +def list_domains(domain: str) -> list[str]: + return [domain, *server.sites.map(lambda s: f"{s.subdomain}.{domain}")] + + +def certbot_command_gen(domain: str, action, test=False) -> str: + domains = list_domains(domain) + match action: + case 'create': + # create with standalone mode + return f'docker run -it --name certbot -v "./data/certbot/certs:/etc/letsencrypt" -v "./data/certbot/data:/var/lib/letsencrypt" certbot/certbot certonly --standalone -d {" -d ".join(domains)}{ " --test-cert" if test else "" }' + case 'renew': + # renew with webroot mode + return f'docker run -it --name certbot -v "./data/certbot/certs:/etc/letsencrypt" -v "./data/certbot/data:/var/lib/letsencrypt" -v "./data/certbot/webroot:/var/www/certbot" certbot/certbot renew --webroot -w /var/www/certbot' + raise ValueError('Invalid action') diff --git a/tool/modules/path.py b/tool/modules/path.py new file mode 100644 index 0000000..94adc27 --- /dev/null +++ b/tool/modules/path.py @@ -0,0 +1,13 @@ +import os.path + +script_dir = os.path.relpath(os.path.dirname(__file__)) +project_dir = os.path.normpath(os.path.join(script_dir, "../../")) +template_dir = os.path.join(project_dir, "template") +nginx_template_dir = os.path.join(template_dir, "nginx") +data_dir = os.path.join(project_dir, "data") +tool_dir = os.path.join(project_dir, "tool") +config_file_path = os.path.join(data_dir, "config") +nginx_config_dir = os.path.join(project_dir, "nginx-config") + +__all__ = ["script_dir", "project_dir", "template_dir", + "nginx_template_dir", "data_dir", "config_file_path", "tool_dir", "nginx_config_dir"] diff --git a/tool/modules/template.py b/tool/modules/template.py new file mode 100644 index 0000000..15238ea --- /dev/null +++ b/tool/modules/template.py @@ -0,0 +1,32 @@ +import os.path +import re + + +class Template: + def __init__(self, template_path: str, var_prefix: str = "CRUPEST"): + if len(var_prefix) != 0 and re.fullmatch(r"^[a-zA-Z_][a-zA-Z0-9_]*$", var_prefix) is None: + raise ValueError("Invalid var prefix.") + self.template_path = template_path + self.template_name = os.path.basename( + template_path)[:-len(".template")] + with open(template_path, "r") as f: + self.template = f.read() + self.var_prefix = var_prefix + self.__var_regex = re.compile(r"\$(" + var_prefix + r"_[a-zA-Z0-9_]+)") + self.__var_brace_regex = re.compile( + r"\$\{\s*(" + var_prefix + r"_[a-zA-Z0-9_]+)\s*\}") + var_set = set() + for match in self.__var_regex.finditer(self.template): + var_set.add(match.group(1)) + for match in self.__var_brace_regex.finditer(self.template): + var_set.add(match.group(1)) + self.var_set = var_set + + def generate(self, config: dict[str, str]) -> str: + result = self.template + for var in self.var_set: + if var not in config: + raise ValueError(f"Missing config var {var}.") + result = result.replace("$" + var, config[var]) + re.sub(r"\$\{\s*" + var + r"\s*\}", config[var], result) + return result diff --git a/tool/setup.py b/tool/setup.py deleted file mode 100755 index 8781fcf..0000000 --- a/tool/setup.py +++ /dev/null @@ -1,377 +0,0 @@ -#!/usr/bin/env python3 - -import os -import os.path -import re -import pwd -import grp -import sys -import argparse -import typing -import shutil -import urllib.request -from rich.console import Console -from rich.prompt import Prompt, Confirm - -console = Console() - - -def print_order(number: int, total: int, *, console=console) -> None: - console.print(f"\[{number}/{total}]", end=" ", style="green") - - -parser = argparse.ArgumentParser( - description="Crupest server all-in-one setup script. Have fun play with it!") -subparsers = parser.add_subparsers(dest="action") - -setup_parser = subparsers .add_parser( - "setup", help="Do everything necessary to setup the server.") - -download_tools_parser = subparsers .add_parser( - "download-tools", help="Download some extra tools to manage the server.") - -clear_parser = subparsers .add_parser( - "clear", help="Delete existing data so you can make a fresh start.") -clear_parser.add_argument("-D", "--include-data-dir", action="store_true", - default=False, help="Also delete the data directory.") - -args = parser.parse_args() - -console.print("Nice to see you! :waving_hand:", style="cyan") - -# get script dir in relative path -script_dir = os.path.relpath(os.path.dirname(__file__)) -project_dir = os.path.normpath(os.path.join(script_dir, "../")) -template_dir = os.path.join(project_dir, "template") -data_dir = os.path.join(project_dir, "data") -tool_dir = os.path.join(project_dir, "tool") - - -def download_tools(): - SCRIPTS = [("docker-mailserver setup script", "docker-mailserver-setup.sh", - "https://raw.githubusercontent.com/docker-mailserver/docker-mailserver/master/setup.sh")] - for index, script in enumerate(SCRIPTS): - number = index + 1 - total = len(SCRIPTS) - print_order(number, total) - name, filename, url = script - path = os.path.join(tool_dir, filename) - skip = False - if os.path.exists(path): - overwrite = Confirm.ask( - f"[cyan]{name}[/] already exists, download and overwrite?", default=False) - if not overwrite: - skip = True - else: - download = Confirm.ask( - f"Download [cyan]{name}[/] to [magenta]{path}[/]?", default=True) - if not download: - skip = True - if not skip: - console.print(f"Downloading {name}...") - urllib.request.urlretrieve(url, path) - os.chmod(path, 0o755) - console.print(f"Downloaded {name} to {path}.", style="green") - else: - console.print(f"Skipped {name}.", style="yellow") - - -if args.action == 'download-tools': - download_tools() - exit(0) - -print("First let's check all the templates...") - -# get all filenames ending with .template -template_name_list = [os.path.basename(f)[:-len('.template')] for f in os.listdir( - template_dir) if f.endswith(".template")] - -# if action is 'clean' -if args.action == "clear": - # check root if we have to delete data dir - if args.include_data_dir and os.path.exists(data_dir) and os.geteuid() != 0: - console.print("You need to be root to delete data dir.", style="red") - sys.exit(1) - - to_delete = Confirm.ask( - "[yellow]Are you sure you want to delete everything? all your data will be lost![/]", default=False) - if to_delete: - files_to_delete = [] - for template_name in template_name_list: - f = os.path.join(project_dir, template_name) - if os.path.exists(f): - files_to_delete.append(f) - - delete_data_dir = args.include_data_dir and os.path.exists(data_dir) - - if len(files_to_delete) == 0: - console.print("Nothing to delete. We are safe!", style="green") - exit(0) - - console.print("Here are the files to delete:") - for f in files_to_delete: - console.print(f, style="magenta") - if delete_data_dir: - console.print(data_dir + " (data dir)", style="magenta") - - to_delete = Confirm.ask( - "[yellow]Are you sure you want to delete them?[/]", default=False) - if to_delete: - for f in files_to_delete: - os.remove(f) - if delete_data_dir: - # recursively delete data dir - shutil.rmtree(data_dir) - console.print( - "Your workspace is clean now! However config file is still there! See you!", style="green") - exit(0) - -console.print( - f"I have found following template files in [magenta]{template_dir}[/]:", style="green") -for filename in template_name_list: - console.print(f"- [magenta]{filename}.template[/]") - - -class ConfigVar: - def __init__(self, name: str, description: str, default_value_generator: typing.Callable[[], str] | str): - """Create a config var. - - Args: - name (str): The name of the config var. - description (str): The description of the config var. - default_value_generator (typing.Callable([], str) | str): The default value generator of the config var. If it is a string, it will be used as the input prompt and let user input the value. - """ - self.name = name - self.description = description - self.default_value_generator = default_value_generator - - def get_default_value(self): - if isinstance(self.default_value_generator, str): - return Prompt.ask(self.default_value_generator, console=console) - else: - return self.default_value_generator() - - -config_var_list: list[ConfigVar] = [ - ConfigVar("CRUPEST_DOMAIN", "domain name", - "Please input your domain name:"), - # ConfigVar("CRUPEST_EMAIL", "admin email address", - # "Please input your email address:"), - ConfigVar("CRUPEST_USER", "your system account username", - lambda: pwd.getpwuid(os.getuid()).pw_name), - ConfigVar("CRUPEST_GROUP", "your system account group name", - lambda: grp.getgrgid(os.getgid()).gr_name), - ConfigVar("CRUPEST_UID", "your system account uid", - lambda: str(os.getuid())), - ConfigVar("CRUPEST_GID", "your system account gid", - lambda: str(os.getgid())), - ConfigVar("CRUPEST_HALO_DB_PASSWORD", - "password for halo h2 database, once used never change it", lambda: os.urandom(8).hex()), - ConfigVar("CRUPEST_IN_CHINA", - "set to true if you are in China, some network optimization will be applied", lambda: "false") -] - -config_var_name_set = set([config_var.name for config_var in config_var_list]) - - -class Template: - - def __init__(self, template_path: str, var_prefix: str = "CRUPEST"): - if len(var_prefix) != 0 and re.fullmatch(r"^[a-zA-Z_][a-zA-Z0-9_]*$", var_prefix) is None: - raise ValueError("Invalid var prefix.") - self.template_path = template_path - self.template_name = os.path.basename( - template_path)[:-len(".template")] - with open(template_path, "r") as f: - self.template = f.read() - self.var_prefix = var_prefix - self.__var_regex = re.compile(r"\$(" + var_prefix + r"_[a-zA-Z0-9_]+)") - self.__var_brace_regex = re.compile( - r"\$\{\s*(" + var_prefix + r"_[a-zA-Z0-9_]+)\s*\}") - var_set = set() - for match in self.__var_regex.finditer(self.template): - var_set.add(match.group(1)) - for match in self.__var_brace_regex.finditer(self.template): - var_set.add(match.group(1)) - self.var_set = var_set - - def generate(self, config: dict[str, str]) -> str: - result = self.template - for var in self.var_set: - if var not in config: - raise ValueError(f"Missing config var {var}.") - result = result.replace("$" + var, config[var]) - re.sub(r"\$\{\s*" + var + r"\s*\}", config[var], result) - return result - - -template_list: list[Template] = [] -config_var_name_set_in_template = set() -for template_path in os.listdir(template_dir): - if not template_path.endswith(".template"): - continue - template = Template(os.path.join(template_dir, template_path)) - template_list.append(template) - config_var_name_set_in_template.update(template.var_set) - -console.print( - "I have found following variables needed in templates:", style="green") -for key in config_var_name_set_in_template: - console.print(key, end=" ", style="magenta") -console.print("") - -# check vars -if not config_var_name_set_in_template == config_var_name_set: - console.print( - "The variables needed in templates are not same to the explicitly declared ones! There must be something wrong.", style="red") - console.print("The explicitly declared ones are:") - for key in config_var_name_set: - console.print(key, end=" ", style="magenta") - console.print( - "\nTry to check template files and edit the var list at the head of this script. Aborted! See you next time!") - exit(1) - - -console.print("Now let's check if they are already generated...") - -conflict = False - -# check if there exists any generated files -for filename in template_name_list: - if os.path.exists(os.path.join(project_dir, filename)): - console.print(f"Found [magenta]{filename}[/]") - conflict = True - -if conflict: - to_overwrite = Confirm.ask( - "It seems there are some files already generated. Do you want to overwrite them?", console=console, default=False) - if not to_overwrite: - console.print( - "Great! Check the existing files and see you next time!", style="green") - exit() -else: - print("No conflict found. Let's go on!\n") - -console.print("Check for existing config file...") - -config_path = os.path.join(data_dir, "config") - - -def parse_config(str: str) -> dict: - config = {} - for line_number, line in enumerate(str.splitlines()): - # check if it's a comment - if line.startswith("#"): - continue - # check if there is a '=' - if line.find("=") == -1: - console.print( - f"Invalid config file. Please check line {line_number + 1}. There is even no '='! Aborted!", style="red") - raise ValueError("Invalid config file.") - # split at first '=' - key, value = line.split("=", 1) - key = key.strip() - value = value.strip() - config[key] = value - return config - - -def config_to_str(config: dict) -> str: - return "\n".join([f"{key}={value}" for key, value in config.items()]) - - -def print_config(config: dict) -> None: - for key, value in config.items(): - console.print(f"[magenta]{key}[/] = [cyan]{value}") - - -# check if there exists a config file -if not os.path.exists(config_path): - config = {} - console.print( - "No existing config file found. Don't worry. Let's create one!", style="green") - for config_var in config_var_list: - config[config_var.name] = config_var.get_default_value() - config_content = config_to_str(config) - # create data dir if not exist - if not os.path.exists(data_dir): - os.mkdir(data_dir) - # write config file - with open(config_path, "w") as f: - f.write(config_content) - console.print( - f"Everything else is auto generated. The config file is written into [magenta]{config_path}[/]. You had better keep it well. And here is the content:", style="green") - print_config(config) - is_ok = Confirm.ask( - "If you think it's not ok, you can stop here and edit it. Or let's go on?", console=console, default=True) - if not is_ok: - console.print( - "Great! Check the config file and see you next time!", style="green") - exit() -else: - console.print( - "Looks like you have already had a config file. Let's check the content:", style="green") - with open(config_path, "r") as f: - content = f.read() - config = parse_config(content) - print_config(config) - missed_config_vars = [] - for config_var in config_var_list: - if config_var.name not in config: - missed_config_vars.append(config_var) - - if len(missed_config_vars) > 0: - console.print( - "Oops! It seems you have missed some keys in your config file. Let's add them!", style="green") - for config_var in missed_config_vars: - config[config_var.name] = config_var.get_default_value() - content = config_to_str(config) - with open(config_path, "w") as f: - f.write(content) - console.print( - f"Here is the new config, it has been written out to [magenta]{config_path}[/]:") - print_config(config) - good_enough = Confirm.ask("Is it good enough?", - console=console, default=True) - if not good_enough: - console.print( - "Great! Check the config file and see you next time!", style="green") - exit() - -console.print( - "Finally, everything is ready. Let's generate the files:", style="green") - -# generate files -for index, template in enumerate(template_list): - number = index + 1 - total = len(template_list) - print_order(number, total) - console.print( - f"Generating [magenta]{template.template_name}[/]...") - content = template.generate(config) - with open(os.path.join(project_dir, filename), "w") as f: - f.write(content) - -if not os.path.exists(os.path.join(data_dir, "code-server")): - os.mkdir(os.path.join(data_dir, "code-server")) - console.print( - "I also create data dir for code-server. Because letting docker create it would result in permission problem.", style="green") -else: - code_server_stat = os.stat(os.path.join(data_dir, "code-server")) - if code_server_stat.st_uid == 0 or code_server_stat.st_gid == 0: - console.print( - "WARNING: The owner of data dir for code-server is root. This may cause permission problem. You had better change it.", style="yellow") - to_fix = Confirm.ask( - "Do you want me to help you fix it?", console=console, default=True) - if to_fix: - os.system( - f"sudo chown -R {os.getuid()}:{os.getgid()} {os.path.join(data_dir, 'code-server')}") - -console.print(":beers: All done!", style="green") -to_download_tools = Confirm.ask( - "By the way, would you like to download some scripts to do some extra setup like creating email user?", console=console, default=True) -if not to_download_tools: - console.print("Great! See you next time!", style="green") - exit() - -download_tools() -- cgit v1.2.3