diff options
| -rwxr-xr-x[l---------] | aio | 3 | ||||
| -rw-r--r-- | docker/crupest-nginx/sites/www/src/style.css | 1 | ||||
| -rwxr-xr-x | tool/aio.py | 821 | ||||
| -rw-r--r-- | tool/modules/backup.py | 41 | ||||
| -rw-r--r-- | tool/modules/check.py | 20 | ||||
| -rw-r--r-- | tool/modules/config.py | 44 | ||||
| -rw-r--r-- | tool/modules/configfile.py | 41 | ||||
| -rw-r--r-- | tool/modules/download_tools.py | 47 | ||||
| -rw-r--r-- | tool/modules/helper.py | 18 | ||||
| -rw-r--r-- | tool/modules/install_docker.py | 16 | ||||
| -rwxr-xr-x | tool/modules/nginx.py | 134 | ||||
| -rw-r--r-- | tool/modules/test.py | 31 | ||||
| -rwxr-xr-x | tool/test-crupest-api.py | 39 |
13 files changed, 658 insertions, 598 deletions
@@ -1 +1,2 @@ -./tool/aio.py
\ No newline at end of file +#! /usr/bin/env sh +exec python3 "$(dirname "$0")/tool/aio.py" "$@" diff --git a/docker/crupest-nginx/sites/www/src/style.css b/docker/crupest-nginx/sites/www/src/style.css index b632745..c692ba7 100644 --- a/docker/crupest-nginx/sites/www/src/style.css +++ b/docker/crupest-nginx/sites/www/src/style.css @@ -6,6 +6,7 @@ body { width: 100%; margin: 0; box-sizing: border-box; + padding: 0 1em; } @media (min-width: 576px) { diff --git a/tool/aio.py b/tool/aio.py index cddd814..e2a9f13 100755 --- a/tool/aio.py +++ b/tool/aio.py @@ -10,20 +10,22 @@ except ImportError: import datetime import os -import os.path -import sys +from os.path import * import argparse import shutil import subprocess -import urllib.request -import re from rich.console import Console from rich.prompt import Confirm +from modules.install_docker import * from modules.path import * from modules.template import Template from modules.nginx import * -from modules.configfile import * from modules.config import * +from modules.check import * +from modules.backup import * +from modules.download_tools import * +from modules.helper import * +from modules.test import * console = Console() @@ -31,6 +33,9 @@ parser = argparse.ArgumentParser( description="Crupest server all-in-one setup script. Have fun play with it!") parser.add_argument("--no-hello", action="store_true", default=False, help="Do not print hello message.") +parser.add_argument("--no-bye-bye", action="store_true", + default=False, help="Do not print bye-bye message.") + parser.add_argument("--no-check-python-version", action="store_true", default=False, help="Do not check python version.") parser.add_argument("--no-check-system", action="store_true", @@ -82,11 +87,15 @@ backup_parser = subparsers.add_parser( "backup", help="Backup related things." ) -backup_command_group = backup_parser.add_mutually_exclusive_group() -backup_command_group.add_argument( - "-R", "--restore", action="append", nargs="?", default=None, help="Restore data from url.") -backup_command_group.add_argument( - "-B", "--backup", action="append", nargs="?", default=None, help="Backup data to specified path.") +backup_subparsers = backup_parser.add_subparsers(dest="backup_action") +backup_restore_parser = backup_subparsers.add_parser( + "restore", help="Restore data from url.") +backup_restore_parser.add_argument( + "restore_url", help="Restore archive url. Can be local path or http/https.") +backup_backup_parser = backup_subparsers.add_parser( + "backup", help="Backup data to specified path.") +backup_backup_parser.add_argument( + "backup_path", nargs="?", help="Backup path. Can be empty for a timestamp as name. Must be local path.") docker_parser = subparsers.add_parser("docker", help="Docker related things.") docker_subparsers = docker_parser.add_subparsers(dest="docker_action") @@ -95,6 +104,10 @@ docker_subparsers.add_parser("down", help="Run docker compose down.") docker_subparsers.add_parser( "prune", help="Run docker system prune -a -f.") +test_parser = subparsers.add_parser("test", help="Test things.") +test_parser.add_argument( + "test_action", help="Test action.", choices=["crupest-api"]) + args = parser.parse_args() if args.yes: @@ -110,293 +123,258 @@ if args.yes: Confirm.ask = new_ask +if args.action == "certbot": + if args.create or args.renew or args.expand: + args.no_hello = True -if not args.no_check_python_version: - if sys.version_info < (3, 10): - console.print("This script works well on python 3.10 or higher. Otherwise you may encounter some problems. But I would like to improve some rational compatibility.", style="yellow") - - -def check_ubuntu(): - if not os.path.exists("/etc/os-release"): - return False - else: - with open("/etc/os-release", "r") as f: - content = f.read() - if re.search(r"NAME=\"?Ubuntu\"?", content, re.IGNORECASE) is None: - return False - if re.search(r"VERSION_ID=\"?22.04\"?", content, re.IGNORECASE) is None: - return False - return True +if not args.no_check_python_version: + if not check_python_version(): + console.print("This script works well on python 3.10. Otherwise you may encounter some problems. But I would like to improve some rational compatibility.", style="yellow") if not args.no_check_system: if not check_ubuntu(): console.print("This script works well on Ubuntu 22.04. Otherwise you may encounter some problems. But I would like to improve some rational compatibility.", style="yellow") -if args.action == "certbot": - if args.create or args.renew or args.expand: - args.no_hello = True if not args.no_hello: console.print("Nice to see you! :waving_hand:", style="cyan") -def print_order(number: int, total: int, *, console=console) -> None: - console.print(f"\[{number}/{total}]", end=" ", style="green") +def check_domain_is_defined(): + try: + return get_domain() + except Exception as e: + console.print(e.args[0], style="red") -if args.action == "install-docker": - ensure_tmp_dir() - get_docker_path = os.path.join(tmp_dir, "get-docker.sh") - urllib.request.urlretrieve("https://get.docker.com", get_docker_path) - os.chmod(get_docker_path, 0o755) - subprocess.run(["sudo", "sh", get_docker_path], check=True) - subprocess.run(["sudo", "systemctl", "enable", - "--now", "docker"], check=True) - subprocess.run(["sudo", "usermod", "-aG", "docker", - os.getlogin()], check=True) - console.print( - "Succeeded to install docker. Please re-login to take effect.", style="green") - exit(0) - - -if args.action == "docker": - def run_in_dir(dir: str, func: callable): - old_dir = os.path.abspath(os.getcwd()) - os.chdir(dir) - func() - os.chdir(old_dir) - match args.docker_action: - case "up": - def docker_compose_up(): - subprocess.run(["docker", "compose", "up", "-d"], check=True) - run_in_dir(project_abs_path, docker_compose_up) - case "down": - def docker_compose_down(): - subprocess.run(["docker", "compose", "down"], check=True) - run_in_dir(project_abs_path, docker_compose_down) - case "prune": - to_do = Confirm.ask("[yellow]Are you sure to prune docker?[/]") - if to_do: - subprocess.run( - ["docker", "system", "prune", "-a", "-f"], check=True) - case _: - raise ValueError("Unknown docker action.") - exit(0) - -if args.action == "backup": - if not args.restore is None: - if args.restore[0] is None: - url = Prompt.ask( - "You don't specify the path to restore from. Please specify one. http and https are supported", console=console) - else: - url = args.restore[0] - if len(url) == 0: - console.print("You specify an empty url. Abort.", style="red") - exit(1) - if url.startswith("http://") or url.startswith("https://"): - download_path = os.path.join(tmp_dir, "data.tar.xz") - if os.path.exists(download_path): - to_remove = Confirm.ask( - f"I want to download to {download_path}. However, there is already a file there. Do you want to remove it first", default=False) - if to_remove: - os.remove(download_path) - else: - console.print( - "Aborted! Please check the file and try again.", style="cyan") - exit(0) - urllib.request.urlretrieve(url, download_path) - url = download_path - subprocess.run( - ["sudo", "tar", "-xJf", url, "-C", project_dir], check=True) - console.print("Succeeded to restore data.", style="green") - exit(0) - elif not args.backup is None: - if args.backup[0] is None: - ensure_backup_dir() - now = datetime.datetime.utcnow().isoformat(timespec="seconds") + "Z" - path = Prompt.ask( - "You don't specify the path to backup to. Please specify one. http and https are NOT supported", console=console, default=os.path.join(backup_dir, now + ".tar.xz")) - else: - path = args.backup[0] - if len(path) == 0: - console.print("You specify an empty path. Abort.", style="red") - exit(1) - if os.path.exists(path): - console.print( - "A file is already there. Please remove it first. Abort!", style="red") - exit(1) - subprocess.run( - ["sudo", "tar", "-cJf", path, "data", "-C", project_dir], - check=True - ) - console.print("Succeeded to backup data.", style="green") - exit(0) - else: +def data_dir_check(domain): + if not exists(data_dir): console.print( - "You should specify either -R or -B. Abort!", style="red") - exit(1) - -if args.action == 'print-path': - console.print("Project path =", project_dir) - console.print("Project absolute path =", project_abs_path) - console.print("Data path =", data_dir) - exit(0) - - -def check_domain_is_defined() -> str: - try: - return get_domain() - except ValueError as e: + "Looks like you haven't generated data dir. I'll create it for you.", style="green") + os.mkdir(data_dir) + elif not isdir(data_dir): console.print( - "We are not able to get the domain. You may want to first run setup command.", style="red") - console.print_exception(e) - exit(1) + "ERROR: data dir is not a dir! Everything will be broken! Please delete it manually", style="red") + if isdir(data_dir): + if not exists(join(data_dir, "certbot")): + print_create_cert_message(domain, console) + else: + to_check = Confirm.ask( + "I want to check your ssl certs, but I need to sudo. Do you want me check", console=console, default=False) + if to_check: + check_ssl_cert() -def download_tools(): - # if we are not linux, we prompt the user - if sys.platform != "linux": + if not exists(join(data_dir, "code-server")): + os.mkdir(join(data_dir, "code-server")) console.print( - "You are not running this script on linux. The tools will not work.", style="yellow") - if not Confirm.ask("Do you want to continue?", default=False, console=console): - exit(0) - - SCRIPTS = [("docker-mailserver setup script", "docker-mailserver-setup.sh", - "https://raw.githubusercontent.com/docker-mailserver/docker-mailserver/master/setup.sh")] - for index, script in enumerate(SCRIPTS): - number = index + 1 - total = len(SCRIPTS) - print_order(number, total) - name, filename, url = script - # if url is callable, call it - if callable(url): - url = url() - path = os.path.join(tool_dir, filename) - skip = False - if os.path.exists(path): - overwrite = Confirm.ask( - f"[cyan]{name}[/] already exists, download and overwrite?", default=False) - if not overwrite: - skip = True - else: - download = Confirm.ask( - f"Download [cyan]{name}[/] to [magenta]{path}[/]?", default=True) - if not download: - skip = True - if not skip: - console.print(f"Downloading {name}...") - urllib.request.urlretrieve(url, path) - os.chmod(path, 0o755) - console.print(f"Downloaded {name} to {path}.", style="green") - else: - console.print(f"Skipped {name}.", style="yellow") + "I also create data dir for code-server. Because letting docker create it would result in permission problem.", style="green") + else: + code_server_stat = os.stat( + join(data_dir, "code-server")) + if code_server_stat.st_uid == 0 or code_server_stat.st_gid == 0: + console.print( + "WARNING: The owner of data dir for code-server is root. This may cause permission problem. You had better change it.", style="yellow") + to_fix = Confirm.ask( + "Do you want me to help you fix it?", console=console, default=True) + if to_fix: + subprocess.run( + ["sudo", "chown", "-R", f"{os.getuid()}:{os.getgid()}", join(data_dir, 'code-server')], check=True) -def generate_nginx_config(domain: str) -> None: - bad_files = nginx_config_dir_check(nginx_config_dir, domain) - if len(bad_files) > 0: - console.print( - "WARNING: It seems there are some bad conf files in the nginx config directory:", style="yellow") - for bad_file in bad_files: - console.print(bad_file, style="cyan") - to_delete = Confirm.ask( - "They will affect nginx in a [red]bad[/] way. Do you want to delete them?", default=True, console=console) - if to_delete: - for file in bad_files: - os.remove(os.path.join(nginx_config_dir, file)) +def setup(template_name_list): + template_list: list = [] + config_var_name_set_in_template = set() + for template_path in os.listdir(template_dir): + if not template_path.endswith(".template"): + continue + template = Template(join( + template_dir, template_path)) + template_list.append(template) + config_var_name_set_in_template.update(template.var_set) + console.print( - "I have found following var in nginx templates:", style="green") - for var in nginx_var_set: - console.print(var, end=" ", style="magenta") - console.print() - if not os.path.exists(nginx_config_dir): - os.mkdir(nginx_config_dir) + "I have found following variables needed in templates:", style="green") + for key in config_var_name_set_in_template: + console.print(key, style="magenta") + + # check vars + check_success, more, less = check_config_var_set( + config_var_name_set_in_template) + if len(more) != 0: + console.print("There are more variables in templates than in config file:", + style="red") + for key in more: + console.print(key, style="magenta") + if len(less) != 0: + console.print("Following config vars are not used:", + style="yellow") + for key in less: + console.print(key, style="magenta") + + if not check_success: console.print( - f"Nginx config directory created at [magenta]{nginx_config_dir}[/]", style="green") - nginx_config_gen(domain, dest=nginx_config_dir) - console.print("Nginx config generated.", style="green") - - -if args.action == 'list-domain': - domain = check_domain_is_defined() - domains = list_domains(domain) - for domain in domains: - console.print(domain) - exit(0) - -if args.action == 'certbot': - domain = check_domain_is_defined() - is_test = args.test - if args.create: - console.print(certbot_command_gen(domain, "create", - test=is_test), soft_wrap=True, highlight=False) - exit(0) - elif args.expand: - console.print(certbot_command_gen(domain, "expand", - test=is_test), soft_wrap=True, highlight=False) - exit(0) - elif args.renew: - console.print(certbot_command_gen(domain, "renew", - test=is_test), soft_wrap=True, highlight=False) - exit(0) - console.print( - "Here is some commands you can use to do certbot related work.") - if is_test: + "Please check you config vars and make sure the needed ones are defined!", style="red") + else: console.print( - "Note you specified --test, so the commands are for test use.", style="yellow") - console.print( - f"To create certs for init:\n[code]{certbot_command_gen(domain, 'create', test=is_test)}[/]") - console.print( - f"To renew certs previously created:\n[code]{certbot_command_gen(domain, 'renew', test=is_test)}[/]") - exit(0) - -if args.action == 'nginx': - domain = check_domain_is_defined() - generate_nginx_config(domain) - exit(0) - + "Now let's check if they are already generated...") + + conflict = False + + # check if there exists any generated files + for filename in template_name_list: + if exists(join(project_dir, filename)): + console.print(f"Found [magenta]{filename}[/]") + conflict = True + + to_gen = True + if conflict: + to_overwrite = Confirm.ask( + "It seems there are some files already generated. Do you want to overwrite them?", console=console, default=False) + if not to_overwrite: + to_gen = False + console.print( + "Great! Check the existing files and see you next time!", style="green") + else: + print("No conflict found. Let's go on!\n") + + if to_gen: + console.print("Check for existing config file...") + + # check if there exists a config file + if not config_file_exists(): + config = {} + console.print( + "No existing config file found. Don't worry. Let's create one!", style="green") + for config_var in config_var_list: + config[config_var.name] = config_var.get_default_value() + config_content = config_to_str(config) + # create data dir if not exist + if not exists(data_dir): + os.mkdir(data_dir) + # write config file + with open(config_file_path, "w") as f: + f.write(config_content) + console.print( + f"Everything else is auto generated. The config file is written into [magenta]{config_file_path}[/]. You had better keep it safe. And here is the content:", style="green") + print_config(console, config) + is_ok = Confirm.ask( + "If you think it's not ok, you can stop here and edit it. Or let's go on?", console=console, default=True) + if not is_ok: + console.print( + "Great! Check the config file and see you next time!", style="green") + to_gen = False + else: + console.print( + "Looks like you have already had a config file. Let's check the content:", style="green") + with open(config_file_path, "r") as f: + content = f.read() + config = parse_config(content) + print_config(console, config) + missed_config_vars = [] + for config_var in config_var_list: + if config_var.name not in config: + missed_config_vars.append(config_var) + + if len(missed_config_vars) > 0: + console.print( + "Oops! It seems you have missed some keys in your config file. Let's add them!", style="green") + for config_var in missed_config_vars: + config[config_var.name] = config_var.get_default_value( + console) + content = config_to_str(config) + with open(config_file_path, "w") as f: + f.write(content) + console.print( + f"Here is the new config, it has been written out to [magenta]{config_file_path}[/]:") + print_config(console, config) + good_enough = Confirm.ask("Is it good enough?", + console=console, default=True) + if not good_enough: + console.print( + "Great! Check the config file and see you next time!", style="green") + to_gen = False -if args.action == 'download-tools': - download_tools() - exit(0) + domain = config["CRUPEST_DOMAIN"] -print("First let's check all the templates...") + if to_gen: + console.print( + "Finally, everything is ready. Let's generate the files:", style="green") + + # generate files + for index, template in enumerate(template_list): + number = index + 1 + total = len(template_list) + print_order(number, total, console) + console.print( + f"Generating [magenta]{template.template_name}[/]...") + content = template.generate(config) + with open(join(project_dir, template.template_name), "w") as f: + f.write(content) + + # generate nginx config + if not exists(nginx_config_dir): + to_gen_nginx_conf = Confirm.ask("It seems you haven't generate nginx config. Do you want to generate it?", + default=True, console=console) + else: + # get the latest time of files in nginx template + template_time = 0 + for path in os.listdir(nginx_template_dir): + template_time = max(template_time, os.stat( + join(nginx_template_dir, path)).st_mtime) + console.print( + f"Nginx template update time: {datetime.fromtimestamp(template_time)}") + + nginx_config_time = 0 + for path in os.listdir(nginx_config_dir): + nginx_config_time = max(nginx_config_time, os.stat( + join(nginx_config_dir, path)).st_mtime) + console.print( + f"Generated nginx template update time: {datetime.fromtimestamp(nginx_config_time)}") + if template_time > nginx_config_time: + to_gen_nginx_conf = Confirm.ask("It seems you have updated the nginx template and not regenerate config. Do you want to regenerate the nginx config?", + default=True, console=console) + else: + to_gen_nginx_conf = Confirm.ask("[yellow]It seems you have already generated nginx config. Do you want to overwrite it?[/]", + default=False, console=console) + if to_gen_nginx_conf: + nginx(domain, console) + data_dir_check(domain) -# get all filenames ending with .template -template_name_list = [os.path.basename(f)[:-len('.template')] for f in os.listdir( - template_dir) if f.endswith(".template")] -# if action is 'clean' -if args.action == "clear": +def clean(template_name_list): # check root if we have to delete data dir - if args.include_data_dir and os.path.exists(data_dir) and os.geteuid() != 0: - console.print("You need to be root to delete data dir.", style="red") - sys.exit(1) + if args.include_data_dir and exists(data_dir) and os.geteuid() != 0: + console.print( + "You need to be root to delete data dir.", style="red") + exit(1) to_delete = Confirm.ask( - "[yellow]Are you sure you want to delete everything? all your data will be lost![/]", default=False) + "[yellow]Are you sure you want to delete everything? all your data will be lost![/]", default=False, console=console) if to_delete: files_to_delete = [] for template_name in template_name_list: - f = os.path.join(project_dir, template_name) - if os.path.exists(f): + f = join(project_dir, template_name) + if exists(f): files_to_delete.append(f) - delete_data_dir = args.include_data_dir and os.path.exists(data_dir) - - if len(files_to_delete) == 0: - console.print("Nothing to delete. We are safe!", style="green") - exit(0) + delete_data_dir = args.include_data_dir and exists( + data_dir) + if len(files_to_delete) == 0: + console.print( + "Nothing to delete. We are safe!", style="green") + else: console.print("Here are the files to delete:") for f in files_to_delete: console.print(f, style="magenta") if delete_data_dir: - console.print(data_dir + " (data dir)", style="magenta") + console.print(data_dir + " (data dir)", + style="magenta") to_delete = Confirm.ask( - "[yellow]Are you sure you want to delete them?[/]", default=False) + "[red]Are you sure you want to delete them?[/]", default=False, console=console) if to_delete: for f in files_to_delete: os.remove(f) @@ -404,237 +382,126 @@ if args.action == "clear": # recursively delete data dir shutil.rmtree(data_dir) console.print( - "Your workspace is clean now! However config file is still there! See you!", style="green") - exit(0) - -console.print( - f"I have found following template files in [magenta]{template_dir}[/]:", style="green") -for filename in template_name_list: - console.print(f"- [magenta]{filename}.template[/]") - -template_list: list = [] -config_var_name_set_in_template = set() -for template_path in os.listdir(template_dir): - if not template_path.endswith(".template"): - continue - template = Template(os.path.join(template_dir, template_path)) - template_list.append(template) - config_var_name_set_in_template.update(template.var_set) - -console.print( - "I have found following variables needed in templates:", style="green") -for key in config_var_name_set_in_template: - console.print(key, end=" ", style="magenta") -console.print("") - -# check vars -check_success, more, less = check_config_var_set( - config_var_name_set_in_template) -if len(more) != 0: - console.print("There are more variables in templates than in config file:", - style="red") - for key in more: - console.print(key, style="magenta") -if len(less) != 0: - console.print("However, following config vars are not used:", - style="yellow") - for key in less: - console.print(key, style="magenta") - -if not check_success: - console.print( - "Please check you config vars and make sure the needed ones are defined!", style="red") - exit(1) - -console.print("Now let's check if they are already generated...") - -conflict = False - -# check if there exists any generated files -for filename in template_name_list: - if os.path.exists(os.path.join(project_dir, filename)): - console.print(f"Found [magenta]{filename}[/]") - conflict = True - -if conflict: - to_overwrite = Confirm.ask( - "It seems there are some files already generated. Do you want to overwrite them?", console=console, default=False) - if not to_overwrite: - console.print( - "Great! Check the existing files and see you next time!", style="green") - exit() -else: - print("No conflict found. Let's go on!\n") - -console.print("Check for existing config file...") - - -# check if there exists a config file -if not config_file_exist: - config = {} - console.print( - "No existing config file found. Don't worry. Let's create one!", style="green") - for config_var in config_var_list: - config[config_var.name] = config_var.get_default_value() - config_content = config_to_str(config) - # create data dir if not exist - if not os.path.exists(data_dir): - os.mkdir(data_dir) - # write config file - with open(config_file_path, "w") as f: - f.write(config_content) - console.print( - f"Everything else is auto generated. The config file is written into [magenta]{config_file_path}[/]. You had better keep it well. And here is the content:", style="green") - print_config(console, config) - is_ok = Confirm.ask( - "If you think it's not ok, you can stop here and edit it. Or let's go on?", console=console, default=True) - if not is_ok: - console.print( - "Great! Check the config file and see you next time!", style="green") - exit() -else: - console.print( - "Looks like you have already had a config file. Let's check the content:", style="green") - with open(config_file_path, "r") as f: - content = f.read() - config = parse_config(content) - print_config(console, config) - missed_config_vars = [] - for config_var in config_var_list: - if config_var.name not in config: - missed_config_vars.append(config_var) - - if len(missed_config_vars) > 0: - console.print( - "Oops! It seems you have missed some keys in your config file. Let's add them!", style="green") - for config_var in missed_config_vars: - config[config_var.name] = config_var.get_default_value(console) - content = config_to_str(config) - with open(config_file_path, "w") as f: - f.write(content) - console.print( - f"Here is the new config, it has been written out to [magenta]{config_file_path}[/]:") - print_config(console, config) - good_enough = Confirm.ask("Is it good enough?", - console=console, default=True) - if not good_enough: - console.print( - "Great! Check the config file and see you next time!", style="green") - exit() + "Your workspace is clean now!", style="green") -console.print( - "Finally, everything is ready. Let's generate the files:", style="green") -# generate files -for index, template in enumerate(template_list): - number = index + 1 - total = len(template_list) - print_order(number, total) - console.print( - f"Generating [magenta]{template.template_name}[/]...") - content = template.generate(config) - with open(os.path.join(project_dir, template.template_name), "w") as f: - f.write(content) - -# generate nginx config -if not os.path.exists(nginx_config_dir): - to_gen_nginx_conf = Confirm.ask("It seems you haven't generate nginx config. Do you want to generate it?", - default=True, console=console) -else: - # get the latest time of files in nginx template - template_time = 0 - for path in os.listdir(nginx_template_dir): - template_time = max(template_time, os.stat( - os.path.join(nginx_template_dir, path)).st_mtime) - console.print( - f"Nginx template update time: {datetime.datetime.fromtimestamp(template_time)}") +action = args.action - nginx_config_time = 0 - for path in os.listdir(nginx_config_dir): - nginx_config_time = max(nginx_config_time, os.stat( - os.path.join(nginx_config_dir, path)).st_mtime) - console.print( - f"Generated nginx template update time: {datetime.datetime.fromtimestamp(nginx_config_time)}") - if template_time > nginx_config_time: - to_gen_nginx_conf = Confirm.ask("It seems you have updated the nginx template and not regenerate config. Do you want to regenerate the nginx config?", - default=True, console=console) - else: - to_gen_nginx_conf = Confirm.ask("[yellow]It seems you have already generated nginx config. Do you want to overwrite it?[/]", - default=False, console=console) -if to_gen_nginx_conf: - domain = config["CRUPEST_DOMAIN"] - generate_nginx_config(domain) -if not os.path.exists(data_dir): - console.print( - "Looks like you haven't generated data dir. I'll create it for you.", style="green") - os.mkdir(data_dir) -elif not os.path.isdir(data_dir): - console.print( - "ERROR: data dir is not a dir! Everything will be broken! Please delete it manually", style="red") +def run(): + match action: + case "install-docker": + install_docker() + console.print( + "Succeeded to install docker. Please re-login to take effect.", style="green") + case "docker": + docker_action = args.docker_action + + match docker_action: + case "up": + def docker_compose_up(): + subprocess.run( + ["docker", "compose", "up", "-d"], check=True) + run_in_dir(project_abs_path, docker_compose_up) + case "down": + def docker_compose_down(): + subprocess.run( + ["docker", "compose", "down"], check=True) + run_in_dir(project_abs_path, docker_compose_down) + case "prune": + to_do = Confirm.ask( + "[yellow]Are you sure to prune docker?[/]", console=console) + if to_do: + subprocess.run( + ["docker", "system", "prune", "-a", "-f"], check=True) + case _: + raise ValueError("Unknown docker action.") + + case "backup": + backup_action = args.backup_action + match backup_action: + case "backup": + backup_backup(args.backup_path, console) + console.print("Succeeded to restore data.", style="green") + case "restore": + backup_restore(args.restore_path, console) + console.print("Succeeded to backup data.", style="green") + + case 'print-path': + console.print("Project path =", project_dir) + console.print("Project absolute path =", project_abs_path) + console.print("Data path =", data_dir) + + case "download-tools": + download_tools(console) + + case "list-domain": + domain = check_domain_is_defined() + domains = list_domains(domain) + for domain in domains: + console.print(domain) + case "nginx": + domain = check_domain_is_defined() + nginx(domain, console) + + case "certbot": + domain = check_domain_is_defined() + is_test = args.test + if args.create: + console.print(certbot_command_gen(domain, "create", + test=is_test), soft_wrap=True, highlight=False) + elif args.expand: + console.print(certbot_command_gen(domain, "expand", + test=is_test), soft_wrap=True, highlight=False) + elif args.renew: + console.print(certbot_command_gen(domain, "renew", + test=is_test), soft_wrap=True, highlight=False) + else: + console.print( + "Here is some commands you can use to do certbot related work.") + if is_test: + console.print( + "Note you specified --test, so the commands are for test use.", style="yellow") + console.print( + "To create certs for init (standalone):", style="cyan") + console.print(certbot_command_gen( + domain, 'create', test=is_test), soft_wrap=True) + console.print("To expand certs (nginx):", style="cyan") + console.print(certbot_command_gen( + domain, 'create', test=is_test), soft_wrap=True) + console.print( + "To renew certs previously created (nginx):", style="cyan") + console.print(certbot_command_gen( + domain, 'renew', test=is_test), soft_wrap=True) + case "test": + match args.test_action: + case "crupest-api": + test_crupest_api(console) + case _: + console.print("Test action invalid.", style="red") + case _: + console.print("First let's check all the templates...") -def print_create_cert_message(domain): - console.print( - "Looks like you haven't run certbot to get the init ssl certificates. You may want to run following code to get one:", style="cyan") - console.print(certbot_command_gen(domain, "create"), - soft_wrap=True, highlight=False) - - -def check_ssl_cert(): - domain = check_domain_is_defined() - cert_path = get_cert_path(domain) - tmp_cert_path = os.path.join(tmp_dir, "fullchain.pem") - console.print("Temporarily copy cert to tmp...", style="yellow") - ensure_tmp_dir() - subprocess.run( - ["sudo", "cp", cert_path, tmp_cert_path], check=True) - subprocess.run(["sudo", "chown", str(os.geteuid()), - tmp_cert_path], check=True) - cert_domains = get_cert_domains(tmp_cert_path, domain) - if cert_domains is None: - print_create_cert_message(domain) - else: - cert_domain_set = set(cert_domains) - domains = set(list_domains(domain)) - if not cert_domain_set == domains: + # get all filenames ending with .template + template_name_list = [basename(f)[:-len('.template')] for f in os.listdir( + template_dir) if f.endswith(".template")] console.print( - "Cert domains are not equal to host domains. Run following command to recreate it.", style="red") - console.print(certbot_command_gen( - domain, "create", standalone=True), soft_wrap=True, highlight=False) - console.print("Remove tmp cert...", style="yellow") - os.remove(tmp_cert_path) + f"I have found following template files in [magenta]{template_dir}[/]:", style="green") + for filename in template_name_list: + console.print(f"{filename}.template", style="magenta") + # if action is 'clean' + if action == "clear": + clean(template_name_list) + else: + setup(template_name_list) + if Confirm.ask( + "By the way, would you like to download some scripts to do some extra setup like creating email user?", console=console, default=True): + download_tools(console) -if os.path.isdir(data_dir): - if not os.path.exists(os.path.join(data_dir, "certbot")): - print_create_cert_message(check_domain_is_defined()) - else: - to_check = Confirm.ask( - "I want to check your ssl certs, but I need to sudo. Do you want me check", console=console, default=False) - if to_check: - check_ssl_cert() - - if not os.path.exists(os.path.join(data_dir, "code-server")): - os.mkdir(os.path.join(data_dir, "code-server")) - console.print( - "I also create data dir for code-server. Because letting docker create it would result in permission problem.", style="green") - else: - code_server_stat = os.stat(os.path.join(data_dir, "code-server")) - if code_server_stat.st_uid == 0 or code_server_stat.st_gid == 0: - console.print( - "WARNING: The owner of data dir for code-server is root. This may cause permission problem. You had better change it.", style="yellow") - to_fix = Confirm.ask( - "Do you want me to help you fix it?", console=console, default=True) - if to_fix: - subprocess.run( - ["sudo", "chown", "-R", f"{os.getuid()}:{os.getgid()}", os.path.join(data_dir, 'code-server')], check=True) -console.print(":beers: All done!", style="green") -to_download_tools = Confirm.ask( - "By the way, would you like to download some scripts to do some extra setup like creating email user?", console=console, default=True) -if not to_download_tools: - console.print("Great! See you next time!", style="green") - exit() +run() -download_tools() +if not args.no_bye_bye: + console.print(":beers: All done! Bye bye!", style="green") diff --git a/tool/modules/backup.py b/tool/modules/backup.py new file mode 100644 index 0000000..7921d0d --- /dev/null +++ b/tool/modules/backup.py @@ -0,0 +1,41 @@ +from .path import * +from rich.prompt import Prompt, Confirm +from urllib.request import urlretrieve +import subprocess +from datetime import datetime + + +def backup_restore(http_url_or_path, /, console): + url = http_url_or_path + if len(url) == 0: + raise Exception("You specify an empty url. Abort.") + if url.startswith("http://") or url.startswith("https://"): + download_path = os.path.join(tmp_dir, "data.tar.xz") + if os.path.exists(download_path): + to_remove = Confirm.ask( + f"I want to download to [cyan]{download_path}[/]. However, there is a file already there. Do you want to remove it first", default=False, console=console) + if to_remove: + os.remove(download_path) + else: + raise Exception( + "Aborted! Please check the file and try again.") + urlretrieve(url, download_path) + url = download_path + subprocess.run(["sudo", "tar", "-xJf", url, "-C", project_dir], check=True) + + +def backup_backup(path, /, console): + ensure_backup_dir() + now = datetime.utcnow().isoformat(timespec="seconds") + "Z" + if path is None: + path = Prompt.ask( + "You don't specify the path to backup to. Please specify one. http and https are NOT supported", console=console, default=os.path.join(backup_dir, now + ".tar.xz")) + if len(path) == 0: + raise Exception("You specify an empty path. Abort!") + if os.path.exists(path): + raise Exception( + "A file is already there. Please remove it first. Abort!") + subprocess.run( + ["sudo", "tar", "-cJf", path, "data", "-C", project_dir], + check=True + ) diff --git a/tool/modules/check.py b/tool/modules/check.py new file mode 100644 index 0000000..2a082f6 --- /dev/null +++ b/tool/modules/check.py @@ -0,0 +1,20 @@ +import sys +import re +from os.path import * + + +def check_python_version(required_version=(3, 10)): + return sys.version_info < required_version + + +def check_ubuntu(): + if not exists("/etc/os-release"): + return False + else: + with open("/etc/os-release", "r") as f: + content = f.read() + if re.search(r"NAME=\"?Ubuntu\"?", content, re.IGNORECASE) is None: + return False + if re.search(r"VERSION_ID=\"?22.04\"?", content, re.IGNORECASE) is None: + return False + return True diff --git a/tool/modules/config.py b/tool/modules/config.py index 37ad996..28b09a3 100644 --- a/tool/modules/config.py +++ b/tool/modules/config.py @@ -1,7 +1,8 @@ -from rich.prompt import Prompt import pwd import grp import os +from rich.prompt import Prompt +from .path import config_file_path class ConfigVar: @@ -73,3 +74,44 @@ def check_config_var_set(needed_config_var_set: set): if var_name not in needed_config_var_set: less.append(var_name) return (True if len(more) == 0 else False, more, less) + + +def config_file_exists(): + return os.path.isfile(config_file_path) + + +def parse_config(str: str) -> dict: + config = {} + for line_number, line in enumerate(str.splitlines()): + # check if it's a comment + if line.startswith("#"): + continue + # check if there is a '=' + if line.find("=") == -1: + raise ValueError( + f"Invalid config string. Please check line {line_number + 1}. There is even no '='!") + # split at first '=' + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + config[key] = value + return config + + +def get_domain() -> str: + if not config_file_exists(): + raise ValueError("Config file not found!") + with open(config_file_path) as f: + config = parse_config(f.read()) + if "CRUPEST_DOMAIN" not in config: + raise ValueError("Domain not found in config file!") + return config["CRUPEST_DOMAIN"] + + +def config_to_str(config: dict) -> str: + return "\n".join([f"{key}={value}" for key, value in config.items()]) + + +def print_config(console, config: dict) -> None: + for key, value in config.items(): + console.print(f"[magenta]{key}[/] = [cyan]{value}") diff --git a/tool/modules/configfile.py b/tool/modules/configfile.py deleted file mode 100644 index 6752e58..0000000 --- a/tool/modules/configfile.py +++ /dev/null @@ -1,41 +0,0 @@ -import os.path -from .path import config_file_path - -config_file_exist = os.path.isfile(config_file_path) - - -def parse_config(str: str) -> dict: - config = {} - for line_number, line in enumerate(str.splitlines()): - # check if it's a comment - if line.startswith("#"): - continue - # check if there is a '=' - if line.find("=") == -1: - raise ValueError( - f"Invalid config string. Please check line {line_number + 1}. There is even no '='!") - # split at first '=' - key, value = line.split("=", 1) - key = key.strip() - value = value.strip() - config[key] = value - return config - - -def get_domain() -> str: - if not config_file_exist: - raise ValueError("Config file not found!") - with open(config_file_path) as f: - config = parse_config(f.read()) - if "CRUPEST_DOMAIN" not in config: - raise ValueError("Domain not found in config file!") - return config["CRUPEST_DOMAIN"] - - -def config_to_str(config: dict) -> str: - return "\n".join([f"{key}={value}" for key, value in config.items()]) - - -def print_config(console, config: dict) -> None: - for key, value in config.items(): - console.print(f"[magenta]{key}[/] = [cyan]{value}") diff --git a/tool/modules/download_tools.py b/tool/modules/download_tools.py new file mode 100644 index 0000000..beb06d4 --- /dev/null +++ b/tool/modules/download_tools.py @@ -0,0 +1,47 @@ +import sys +from os.path import * +from urllib.request import * +from rich.prompt import Confirm +from .path import * +from .helper import print_order + + +TOOLS = [("docker-mailserver setup script", "docker-mailserver-setup.sh", + "https://raw.githubusercontent.com/docker-mailserver/docker-mailserver/master/setup.sh")] + + +def download_tools(console): + # if we are not linux, we prompt the user + if sys.platform != "linux": + console.print( + "You are not running this script on linux. The tools will not work.", style="yellow") + if not Confirm.ask("Do you want to continue?", default=False, console=console): + return + + for index, script in enumerate(TOOLS): + number = index + 1 + total = len(TOOLS) + print_order(number, total, console) + name, filename, url = script + # if url is callable, call it + if callable(url): + url = url() + path = join(tool_dir, filename) + skip = False + if exists(path): + overwrite = Confirm.ask( + f"[cyan]{name}[/] already exists, download and overwrite?", default=False, console=console) + if not overwrite: + skip = True + else: + download = Confirm.ask( + f"Download [cyan]{name}[/] to [magenta]{path}[/]?", default=True, console=console) + if not download: + skip = True + if not skip: + console.print(f"Downloading {name}...") + urlretrieve(url, path) + os.chmod(path, 0o755) + console.print(f"Downloaded {name} to {path}.", style="green") + else: + console.print(f"Skipped {name}.", style="yellow") diff --git a/tool/modules/helper.py b/tool/modules/helper.py new file mode 100644 index 0000000..f8fe34a --- /dev/null +++ b/tool/modules/helper.py @@ -0,0 +1,18 @@ +import os +import os.path +from .path import * + + +def run_in_dir(dir: str, func: callable): + old_dir = os.path.abspath(os.getcwd()) + os.chdir(dir) + func() + os.chdir(old_dir) + + +def run_in_project_dir(func: callable): + run_in_dir(project_dir, func) + + +def print_order(number: int, total: int, /, console) -> None: + console.print(f"\[{number}/{total}]", end=" ", style="green") diff --git a/tool/modules/install_docker.py b/tool/modules/install_docker.py new file mode 100644 index 0000000..ac50290 --- /dev/null +++ b/tool/modules/install_docker.py @@ -0,0 +1,16 @@ +from os.path import * +from .path import * +import urllib +import subprocess + + +def install_docker(): + ensure_tmp_dir() + get_docker_path = join(tmp_dir, "get-docker.sh") + urllib.request.urlretrieve("https://get.docker.com", get_docker_path) + os.chmod(get_docker_path, 0o755) + subprocess.run(["sudo", "sh", get_docker_path], check=True) + subprocess.run(["sudo", "systemctl", "enable", + "--now", "docker"], check=True) + subprocess.run(["sudo", "usermod", "-aG", "docker", + os.getlogin()], check=True) diff --git a/tool/modules/nginx.py b/tool/modules/nginx.py index 9c51d66..087422b 100755 --- a/tool/modules/nginx.py +++ b/tool/modules/nginx.py @@ -1,56 +1,65 @@ #!/usr/bin/env python3 -from .template import Template -from .path import * import json import jsonschema import os -import os.path +from os.path import * import shutil +import subprocess +from rich.prompt import Confirm from cryptography.x509 import * from cryptography.x509.oid import ExtensionOID +from .template import Template +from .path import * - -with open(os.path.join(nginx_template_dir, 'server.json')) as f: +with open(join(nginx_template_dir, 'server.json')) as f: server = json.load(f) -with open(os.path.join(nginx_template_dir, 'server.schema.json')) as f: +with open(join(nginx_template_dir, 'server.schema.json')) as f: schema = json.load(f) jsonschema.validate(server, schema) non_template_files = ['forbid_unknown_domain.conf', "websocket.conf"] -ssl_template = Template(os.path.join(nginx_template_dir, 'ssl.conf.template')) -root_template = Template(os.path.join( +ssl_template = Template(join(nginx_template_dir, 'ssl.conf.template')) +root_template = Template(join( nginx_template_dir, 'root.conf.template')) -static_file_template = Template(os.path.join( +static_file_template = Template(join( nginx_template_dir, 'static-file.conf.template')) -reverse_proxy_template = Template(os.path.join( +reverse_proxy_template = Template(join( nginx_template_dir, 'reverse-proxy.conf.template')) -redirect_template = Template(os.path.join( +redirect_template = Template(join( nginx_template_dir, 'redirect.conf.template')) -cert_only_template = Template(os.path.join( +cert_only_template = Template(join( nginx_template_dir, 'cert-only.conf.template')) nginx_var_set = set.union(root_template.var_set, static_file_template.var_set, reverse_proxy_template.var_set) -def nginx_config_gen(domain: str, dest: str) -> None: - if not os.path.isdir(dest): +def list_subdomains(domain: str) -> list: + return [f"{s['subdomain']}.{domain}" for s in server["sites"]] + + +def list_domains(domain: str) -> list: + return [domain, *list_subdomains(domain)] + + +def generate_nginx_config(domain: str, dest: str) -> None: + if not isdir(dest): raise ValueError('dest must be a directory') # copy ssl.conf and https-redirect.conf which need no variable substitution for filename in non_template_files: - src = os.path.join(nginx_template_dir, filename) - dst = os.path.join(dest, filename) + src = join(nginx_template_dir, filename) + dst = join(dest, filename) shutil.copyfile(src, dst) config = {"CRUPEST_DOMAIN": domain} # generate ssl.conf - with open(os.path.join(dest, 'ssl.conf'), 'w') as f: + with open(join(dest, 'ssl.conf'), 'w') as f: f.write(ssl_template.generate(config)) # generate root.conf - with open(os.path.join(dest, f'{domain}.conf'), 'w') as f: + with open(join(dest, f'{domain}.conf'), 'w') as f: f.write(root_template.generate(config)) # generate nginx config for each site sites: list = server["sites"] @@ -72,16 +81,45 @@ def nginx_config_gen(domain: str, dest: str) -> None: template = cert_only_template else: raise Exception('Invalid site type') - with open(os.path.join(dest, f'{subdomain}.{domain}.conf'), 'w') as f: + with open(join(dest, f'{subdomain}.{domain}.conf'), 'w') as f: f.write(template.generate(local_config)) -def list_subdomains(domain: str) -> list: - return [f"{s['subdomain']}.{domain}" for s in server["sites"]] +def check_nginx_config_dir(dir_path: str, domain: str) -> list: + if not exists(dir_path): + return [] + good_files = [*non_template_files, "ssl.conf", * + [f"{full_domain}.conf" for full_domain in list_domains(domain)]] + bad_files = [] + for path in os.listdir(dir_path): + file_name = basename(path) + if file_name not in good_files: + bad_files.append(file_name) + return bad_files -def list_domains(domain: str) -> list: - return [domain, *list_subdomains(domain)] +def nginx(domain: str, /, console) -> None: + bad_files = check_nginx_config_dir(nginx_config_dir, domain) + if len(bad_files) > 0: + console.print( + "WARNING: It seems there are some bad conf files in the nginx config directory:", style="yellow") + for bad_file in bad_files: + console.print(bad_file, style="cyan") + to_delete = Confirm.ask( + "They will affect nginx in a [red]bad[/] way. Do you want to delete them?", default=True, console=console) + if to_delete: + for file in bad_files: + os.remove(join(nginx_config_dir, file)) + console.print( + "I have found following var in nginx templates:", style="green") + for var in nginx_var_set: + console.print(var, style="magenta") + if not exists(nginx_config_dir): + os.mkdir(nginx_config_dir) + console.print( + f"Nginx config directory created at [magenta]{nginx_config_dir}[/]", style="green") + generate_nginx_config(domain, dest=nginx_config_dir) + console.print("Nginx config generated.", style="green") def certbot_command_gen(domain: str, action, /, test=False, no_docker=False, *, standalone=None, email=None, agree_tos=False) -> str: @@ -133,29 +171,16 @@ def certbot_command_gen(domain: str, action, /, test=False, no_docker=False, *, return command -def nginx_config_dir_check(dir_path: str, domain: str) -> list: - if not os.path.exists(dir_path): - return [] - good_files = [*non_template_files, "ssl.conf", * - [f"{full_domain}.conf" for full_domain in list_domains(domain)]] - bad_files = [] - for path in os.listdir(dir_path): - basename = os.path.basename(path) - if basename not in good_files: - bad_files.append(basename) - return bad_files - - def get_cert_path(root_domain): - return os.path.join(data_dir, "certbot", "certs", "live", root_domain, "fullchain.pem") + return join(data_dir, "certbot", "certs", "live", root_domain, "fullchain.pem") def get_cert_domains(cert_path, root_domain): - if not os.path.exists(cert_path): + if not exists(cert_path): return None - if not os.path.isfile(cert_path): + if not isfile(cert_path): return None with open(cert_path, 'rb') as f: @@ -166,3 +191,34 @@ def get_cert_domains(cert_path, root_domain): domains.remove(root_domain) domains = [root_domain, *domains] return domains + + +def print_create_cert_message(domain, console): + console.print( + "Looks like you haven't run certbot to get the init ssl certificates. You may want to run following code to get one:", style="cyan") + console.print(certbot_command_gen(domain, "create"), + soft_wrap=True, highlight=False) + + +def check_ssl_cert(domain, console): + cert_path = get_cert_path(domain) + tmp_cert_path = join(tmp_dir, "fullchain.pem") + console.print("Temporarily copy cert to tmp...", style="yellow") + ensure_tmp_dir() + subprocess.run( + ["sudo", "cp", cert_path, tmp_cert_path], check=True) + subprocess.run(["sudo", "chown", str(os.geteuid()), + tmp_cert_path], check=True) + cert_domains = get_cert_domains(tmp_cert_path, domain) + if cert_domains is None: + print_create_cert_message(domain, console) + else: + cert_domain_set = set(cert_domains) + domains = set(list_domains(domain)) + if not cert_domain_set == domains: + console.print( + "Cert domains are not equal to host domains. Run following command to recreate it with nginx stopped.", style="red") + console.print(certbot_command_gen( + domain, "create", standalone=True), soft_wrap=True, highlight=False) + console.print("Remove tmp cert...", style="yellow") + os.remove(tmp_cert_path) diff --git a/tool/modules/test.py b/tool/modules/test.py new file mode 100644 index 0000000..d6eb778 --- /dev/null +++ b/tool/modules/test.py @@ -0,0 +1,31 @@ +import json +from http.client import * +from urllib.request import urlopen + + +def test_crupest_api(console): + def do_the_test(): + res: HTTPResponse = urlopen("http://localhost:5188/api/todos") + body = res.read() + + if res.status != 200: + raise Exception("Status code is not 200.") + result = json.loads(body) + if not isinstance(result, list): + raise Exception("Result is not an array.") + if len(result) == 0: + raise Exception("Result is an empty array.") + if not isinstance(result[0], dict): + raise Exception("Result[0] is not an object.") + if not isinstance(result[0].get("title"), str): + raise Exception("Result[0].title is not a string.") + if not isinstance(result[0].get("status"), str): + raise Exception("Result[0].status is not a string.") + + try: + do_the_test() + console.print("Test passed!", style="green") + exit(0) + except Exception as e: + console.print(e) + console.print("Test failed!", style="red") diff --git a/tool/test-crupest-api.py b/tool/test-crupest-api.py deleted file mode 100755 index c89a0f9..0000000 --- a/tool/test-crupest-api.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 - -import json -import time -from os.path import * -from urllib.request import urlopen -from http.client import * -from rich.console import Console -from modules.path import * - -console = console = Console() - - -def do_the_test(): - res: HTTPResponse = urlopen("http://localhost:5188/api/todos") - body = res.read() - - if res.status != 200: - raise Exception("Status code is not 200.") - result = json.loads(body) - if not isinstance(result, list): - raise Exception("Result is not an array.") - if len(result) == 0: - raise Exception("Result is an empty array.") - if not isinstance(result[0], dict): - raise Exception("Result[0] is not an object.") - if not isinstance(result[0].get("title"), str): - raise Exception("Result[0].title is not a string.") - if not isinstance(result[0].get("status"), str): - raise Exception("Result[0].status is not a string.") - - -try: - do_the_test() - console.print("Test passed!", style="green") - exit(0) -except Exception as e: - console.print(e) - console.print("Test failed!", style="red") |