diff options
-rw-r--r-- | .gitignore | 2 | ||||
-rw-r--r-- | docker/auto-backup/AutoBackup/.dockerignore | 2 | ||||
-rw-r--r-- | docker/crupest-api/CrupestApi/.dockerignore | 2 | ||||
-rw-r--r-- | docker/crupest-api/CrupestApi/.gitignore | 2 | ||||
-rw-r--r-- | docker/crupest-api/CrupestApi/Config/TodosConfiguration.cs | 15 | ||||
-rw-r--r-- | docker/crupest-api/CrupestApi/CrupestApi.csproj | 8 | ||||
-rw-r--r-- | docker/crupest-api/CrupestApi/Program.cs | 160 | ||||
-rw-r--r-- | docker/crupest-api/CrupestApi/Properties/launchSettings.json | 13 | ||||
-rw-r--r-- | docker/crupest-api/CrupestApi/appsettings.json | 8 | ||||
-rw-r--r-- | docker/crupest-api/Dockerfile | 13 | ||||
-rw-r--r-- | template/crupest-api-config.json.template | 8 | ||||
-rw-r--r-- | template/nginx/reverse-proxy.conf.template | 1 | ||||
-rw-r--r-- | template/nginx/root.conf.template | 15 | ||||
-rwxr-xr-x | tool/aio.py | 79 | ||||
-rw-r--r-- | tool/modules/config.py | 75 | ||||
-rw-r--r-- | tool/modules/path.py | 6 | ||||
-rwxr-xr-x | tool/test-crupest-api.py | 80 |
17 files changed, 426 insertions, 63 deletions
@@ -1,5 +1,7 @@ docker-compose.yaml mailserver.env +crupest-api-config.json data nginx-config +log __pycache__ diff --git a/docker/auto-backup/AutoBackup/.dockerignore b/docker/auto-backup/AutoBackup/.dockerignore new file mode 100644 index 0000000..7de5508 --- /dev/null +++ b/docker/auto-backup/AutoBackup/.dockerignore @@ -0,0 +1,2 @@ +obj +bin diff --git a/docker/crupest-api/CrupestApi/.dockerignore b/docker/crupest-api/CrupestApi/.dockerignore new file mode 100644 index 0000000..7de5508 --- /dev/null +++ b/docker/crupest-api/CrupestApi/.dockerignore @@ -0,0 +1,2 @@ +obj +bin diff --git a/docker/crupest-api/CrupestApi/.gitignore b/docker/crupest-api/CrupestApi/.gitignore new file mode 100644 index 0000000..7de5508 --- /dev/null +++ b/docker/crupest-api/CrupestApi/.gitignore @@ -0,0 +1,2 @@ +obj +bin diff --git a/docker/crupest-api/CrupestApi/Config/TodosConfiguration.cs b/docker/crupest-api/CrupestApi/Config/TodosConfiguration.cs new file mode 100644 index 0000000..68f893a --- /dev/null +++ b/docker/crupest-api/CrupestApi/Config/TodosConfiguration.cs @@ -0,0 +1,15 @@ +using System.ComponentModel.DataAnnotations; + +namespace CrupestApi.Config +{ + public class TodoConfiguration + { + [Required] + public string Username { get; set; } = default!; + [Required] + public int ProjectNumber { get; set; } = default!; + [Required] + public string Token { get; set; } = default!; + public int? Count { get; set; } + } +}
\ No newline at end of file diff --git a/docker/crupest-api/CrupestApi/CrupestApi.csproj b/docker/crupest-api/CrupestApi/CrupestApi.csproj new file mode 100644 index 0000000..4348979 --- /dev/null +++ b/docker/crupest-api/CrupestApi/CrupestApi.csproj @@ -0,0 +1,8 @@ +<Project Sdk="Microsoft.NET.Sdk.Web">
+
+ <PropertyGroup>
+ <TargetFramework>net7.0</TargetFramework>
+ <Nullable>enable</Nullable>
+ </PropertyGroup>
+
+</Project>
diff --git a/docker/crupest-api/CrupestApi/Program.cs b/docker/crupest-api/CrupestApi/Program.cs new file mode 100644 index 0000000..c62bf4d --- /dev/null +++ b/docker/crupest-api/CrupestApi/Program.cs @@ -0,0 +1,160 @@ +using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Text.Json;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Net.Mime;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Console;
+using Microsoft.AspNetCore.Http;
+using CrupestApi.Config;
+
+public class TodoItem
+{
+ public string Status { get; set; } = default!;
+ public string Title { get; set; } = default!;
+}
+
+internal class Program
+{
+ private static void Main(string[] args)
+ {
+ using var httpClient = new HttpClient();
+
+ var builder = WebApplication.CreateBuilder(args);
+
+ string configFilePath = Environment.GetEnvironmentVariable("CRUPEST_API_CONFIG_FILE") ?? "/config.json";
+
+ builder.Configuration.AddJsonFile(configFilePath, optional: false, reloadOnChange: true);
+
+ string? logFilePath = Environment.GetEnvironmentVariable("CRUPEST_API_LOG_FILE");
+ if (logFilePath is not null)
+ {
+ // TODO: Log to file.
+ builder.Logging.AddSimpleConsole(logger =>
+ {
+ logger.ColorBehavior = LoggerColorBehavior.Disabled;
+ });
+ }
+
+ var app = builder.Build();
+
+ app.MapGet("/api/todos", async ([FromServices] IConfiguration configuration, [FromServices] ILoggerFactory loggerFactory) =>
+ {
+ var logger = loggerFactory.CreateLogger("CrupestApi.Todos");
+
+ static string CreateGraphQLQuery(TodoConfiguration todoConfiguration)
+ {
+ return $$"""
+{
+ user(login: "{{todoConfiguration.Username}}") {
+ projectV2(number: {{todoConfiguration.ProjectNumber}}) {
+ items(last: {{todoConfiguration.Count ?? 20}}) {
+ nodes {
+ __typename
+ content {
+ __typename
+ ... on Issue {
+ title
+ closed
+ }
+ ... on PullRequest {
+ title
+ closed
+ }
+ ... on DraftIssue {
+ title
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+""";
+ }
+
+ var todoConfiguration = configuration.GetSection("Todos").Get<TodoConfiguration>();
+ if (todoConfiguration is null)
+ {
+ throw new Exception("Fail to get todos configuration.");
+ }
+
+ using var requestContent = new StringContent(JsonSerializer.Serialize(new
+ {
+ query = CreateGraphQLQuery(todoConfiguration)
+ }));
+ requestContent.Headers.ContentType = new MediaTypeHeaderValue(MediaTypeNames.Application.Json, Encoding.UTF8.WebName);
+
+ using var request = new HttpRequestMessage(HttpMethod.Post, "https://api.github.com/graphql");
+ request.Content = requestContent;
+ request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", todoConfiguration.Token);
+
+ using var response = await httpClient.SendAsync(request);
+ var responseBody = await response.Content.ReadAsStringAsync();
+ logger.LogInformation(response.StatusCode.ToString());
+ logger.LogInformation(responseBody);
+
+
+ if (response.IsSuccessStatusCode)
+ {
+ using var responseJson = JsonSerializer.Deserialize<JsonDocument>(responseBody);
+ if (responseJson is null)
+ {
+ throw new Exception("Fail to deserialize response body.");
+ }
+
+ var nodes = responseJson.RootElement.GetProperty("data").GetProperty("user").GetProperty("projectV2").GetProperty("items").GetProperty("nodes").EnumerateArray();
+
+ var result = new List<TodoItem>();
+
+ foreach (var node in nodes)
+ {
+ var content = node.GetProperty("content");
+ var title = content.GetProperty("title").GetString();
+ if (title is null)
+ {
+ throw new Exception("Fail to get title.");
+ }
+ JsonElement closedElement;
+ bool closed;
+ if (content.TryGetProperty("closed", out closedElement))
+ {
+ closed = closedElement.GetBoolean();
+ }
+ else
+ {
+ closed = false;
+ }
+
+ result.Add(new TodoItem
+ {
+ Title = title,
+ Status = closed ? "Done" : "Todo"
+ });
+ }
+
+ return Results.Json(result, new JsonSerializerOptions
+ {
+ PropertyNamingPolicy = JsonNamingPolicy.CamelCase
+ }, statusCode: 200);
+ }
+ else
+ {
+ const string message = "Fail to get todos from GitHub.";
+ logger.LogError(message);
+
+ return Results.Json(new
+ {
+ message
+ }, statusCode: StatusCodes.Status503ServiceUnavailable);
+ }
+ });
+
+ app.Run();
+ }
+}
\ No newline at end of file diff --git a/docker/crupest-api/CrupestApi/Properties/launchSettings.json b/docker/crupest-api/CrupestApi/Properties/launchSettings.json new file mode 100644 index 0000000..01f2a12 --- /dev/null +++ b/docker/crupest-api/CrupestApi/Properties/launchSettings.json @@ -0,0 +1,13 @@ +{
+ "$schema": "https://json.schemastore.org/launchsettings.json",
+ "profiles": {
+ "dev": {
+ "commandName": "Project",
+ "dotnetRunMessages": true,
+ "applicationUrl": "http://localhost:5188",
+ "environmentVariables": {
+ "ASPNETCORE_ENVIRONMENT": "Development"
+ }
+ }
+ }
+}
\ No newline at end of file diff --git a/docker/crupest-api/CrupestApi/appsettings.json b/docker/crupest-api/CrupestApi/appsettings.json new file mode 100644 index 0000000..53753bd --- /dev/null +++ b/docker/crupest-api/CrupestApi/appsettings.json @@ -0,0 +1,8 @@ +{
+ "Logging": {
+ "LogLevel": {
+ "Default": "Information"
+ }
+ },
+ "AllowedHosts": "*"
+}
\ No newline at end of file diff --git a/docker/crupest-api/Dockerfile b/docker/crupest-api/Dockerfile new file mode 100644 index 0000000..ba7d75b --- /dev/null +++ b/docker/crupest-api/Dockerfile @@ -0,0 +1,13 @@ +FROM mcr.microsoft.com/dotnet/sdk:7.0-alpine AS build +COPY CrupestApi /CrupestApi +WORKDIR /CrupestApi +RUN dotnet publish CrupestApi.csproj --configuration Release --output ./publish/ -r linux-x64 --self-contained false + +FROM mcr.microsoft.com/dotnet/aspnet:7.0-alpine +ENV ASPNETCORE_URLS=http://0.0.0.0:5000 +ENV ASPNETCORE_FORWARDEDHEADERS_ENABLED=true +COPY --from=build /CrupestApi/publish /CrupestApi +WORKDIR /CrupestApi +VOLUME [ "/config.json" ] +EXPOSE 5000 +ENTRYPOINT ["dotnet", "CrupestApi.dll"] diff --git a/template/crupest-api-config.json.template b/template/crupest-api-config.json.template new file mode 100644 index 0000000..fc5eb85 --- /dev/null +++ b/template/crupest-api-config.json.template @@ -0,0 +1,8 @@ +{ + "Todos": { + "Username": "$CRUPEST_GITHUB_USERNAME", + "ProjectNumber": "$CRUPEST_GITHUB_PROJECT_NUMBER", + "Token": "$CRUPEST_GITHUB_TOKEN", + "Count": "$CRUPEST_GITHUB_TODO_COUNT" + } +} diff --git a/template/nginx/reverse-proxy.conf.template b/template/nginx/reverse-proxy.conf.template index 81ba1e9..99184e0 100644 --- a/template/nginx/reverse-proxy.conf.template +++ b/template/nginx/reverse-proxy.conf.template @@ -8,7 +8,6 @@ server { server_name ${CRUPEST_NGINX_SUBDOMAIN}.${CRUPEST_DOMAIN}; location / { - proxy_cache off; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; diff --git a/template/nginx/root.conf.template b/template/nginx/root.conf.template index 8214bf7..110def4 100644 --- a/template/nginx/root.conf.template +++ b/template/nginx/root.conf.template @@ -1,9 +1,24 @@ +upstream crupest-api { + server crupest-api:5000 +} + server { listen 443 ssl http2; listen [::]:443 ssl http2; server_name ${CRUPEST_DOMAIN}; root /srv/www; + + location /api { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_pass http://crupest-api; + } } server { diff --git a/tool/aio.py b/tool/aio.py index 8ccaab9..5498141 100755 --- a/tool/aio.py +++ b/tool/aio.py @@ -2,8 +2,6 @@ import os import os.path -import pwd -import grp import sys import argparse import shutil @@ -14,6 +12,7 @@ from modules.path import * from modules.template import Template from modules.nginx import * from modules.configfile import * +from modules.config import * console = Console() @@ -248,57 +247,6 @@ console.print( for filename in template_name_list: console.print(f"- [magenta]{filename}.template[/]") - -class ConfigVar: - def __init__(self, name: str, description: str, default_value_generator, /, default_value_for_ask=None): - """Create a config var. - - Args: - name (str): The name of the config var. - description (str): The description of the config var. - default_value_generator (typing.Callable([], str) | str): The default value generator of the config var. If it is a string, it will be used as the input prompt and let user input the value. - """ - self.name = name - self.description = description - self.default_value_generator = default_value_generator - self.default_value_for_ask = default_value_for_ask - - def get_default_value(self): - if isinstance(self.default_value_generator, str): - return Prompt.ask(self.default_value_generator, console=console, default=self.default_value_for_ask) - else: - return self.default_value_generator() - - -config_var_list: list = [ - ConfigVar("CRUPEST_DOMAIN", "domain name", - "Please input your domain name"), - ConfigVar("CRUPEST_EMAIL", "admin email address", - "Please input your email address"), - ConfigVar("CRUPEST_USER", "your system account username", - lambda: pwd.getpwuid(os.getuid()).pw_name), - ConfigVar("CRUPEST_GROUP", "your system account group name", - lambda: grp.getgrgid(os.getgid()).gr_name), - ConfigVar("CRUPEST_UID", "your system account uid", - lambda: str(os.getuid())), - ConfigVar("CRUPEST_GID", "your system account gid", - lambda: str(os.getgid())), - ConfigVar("CRUPEST_HALO_DB_PASSWORD", - "password for halo h2 database, once used never change it", lambda: os.urandom(8).hex()), - ConfigVar("CRUPEST_IN_CHINA", - "set to true if you are in China, some network optimization will be applied", lambda: "false"), - ConfigVar("CRUPEST_AUTO_BACKUP_COS_SECRET_ID", - "access key id for Tencent COS, used for auto backup", "Please input your Tencent COS access key id for backup"), - ConfigVar("CRUPEST_AUTO_BACKUP_COS_SECRET_KEY", - "access key secret for Tencent COS, used for auto backup", "Please input your Tencent COS access key for backup"), - ConfigVar("CRUPEST_AUTO_BACKUP_COS_REGION", - "region for Tencent COS, used for auto backup", "Please input your Tencent COS region for backup", "ap-hongkong"), - ConfigVar("CRUPEST_AUTO_BACKUP_BUCKET_NAME", - "bucket name for Tencent COS, used for auto backup", "Please input your Tencent COS bucket name for backup") -] - -config_var_name_set = set([config_var.name for config_var in config_var_list]) - template_list: list = [] config_var_name_set_in_template = set() for template_path in os.listdir(template_dir): @@ -315,17 +263,24 @@ for key in config_var_name_set_in_template: console.print("") # check vars -if not config_var_name_set_in_template == config_var_name_set: - console.print( - "The variables needed in templates are not same to the explicitly declared ones! There must be something wrong.", style="red") - console.print("The explicitly declared ones are:") - for key in config_var_name_set: - console.print(key, end=" ", style="magenta") +check_success, more, less = check_config_var_set( + config_var_name_set_in_template) +if len(more) != 0: + console.print("There are more variables in templates than in config file:", + style="red") + for key in more: + console.print(key, style="magenta") +if len(less) != 0: + console.print("However, following config vars are not used:", + style="yellow") + for key in less: + console.print(key, style="magenta") + +if not check_success: console.print( - "\nTry to check template files and edit the var list at the head of this script. Aborted! See you next time!") + "Please check you config vars and make sure the needed ones are defined!", style="red") exit(1) - console.print("Now let's check if they are already generated...") conflict = False @@ -388,7 +343,7 @@ else: console.print( "Oops! It seems you have missed some keys in your config file. Let's add them!", style="green") for config_var in missed_config_vars: - config[config_var.name] = config_var.get_default_value() + config[config_var.name] = config_var.get_default_value(console) content = config_to_str(config) with open(config_file_path, "w") as f: f.write(content) diff --git a/tool/modules/config.py b/tool/modules/config.py new file mode 100644 index 0000000..37ad996 --- /dev/null +++ b/tool/modules/config.py @@ -0,0 +1,75 @@ +from rich.prompt import Prompt +import pwd +import grp +import os + + +class ConfigVar: + def __init__(self, name: str, description: str, default_value_generator, /, default_value_for_ask=None): + """Create a config var. + + Args: + name (str): The name of the config var. + description (str): The description of the config var. + default_value_generator (typing.Callable([], str) | str): The default value generator of the config var. If it is a string, it will be used as the input prompt and let user input the value. + """ + self.name = name + self.description = description + self.default_value_generator = default_value_generator + self.default_value_for_ask = default_value_for_ask + + def get_default_value(self, /, console): + if isinstance(self.default_value_generator, str): + return Prompt.ask(self.default_value_generator, console=console, default=self.default_value_for_ask) + else: + return self.default_value_generator() + + +config_var_list: list = [ + ConfigVar("CRUPEST_DOMAIN", "domain name", + "Please input your domain name"), + ConfigVar("CRUPEST_EMAIL", "admin email address", + "Please input your email address"), + ConfigVar("CRUPEST_USER", "your system account username", + lambda: pwd.getpwuid(os.getuid()).pw_name), + ConfigVar("CRUPEST_GROUP", "your system account group name", + lambda: grp.getgrgid(os.getgid()).gr_name), + ConfigVar("CRUPEST_UID", "your system account uid", + lambda: str(os.getuid())), + ConfigVar("CRUPEST_GID", "your system account gid", + lambda: str(os.getgid())), + ConfigVar("CRUPEST_HALO_DB_PASSWORD", + "password for halo h2 database, once used never change it", lambda: os.urandom(8).hex()), + ConfigVar("CRUPEST_IN_CHINA", + "set to true if you are in China, some network optimization will be applied", lambda: "false"), + ConfigVar("CRUPEST_AUTO_BACKUP_COS_SECRET_ID", + "access key id for Tencent COS, used for auto backup", "Please input your Tencent COS access key id for backup"), + ConfigVar("CRUPEST_AUTO_BACKUP_COS_SECRET_KEY", + "access key secret for Tencent COS, used for auto backup", "Please input your Tencent COS access key for backup"), + ConfigVar("CRUPEST_AUTO_BACKUP_COS_REGION", + "region for Tencent COS, used for auto backup", "Please input your Tencent COS region for backup", "ap-hongkong"), + ConfigVar("CRUPEST_AUTO_BACKUP_BUCKET_NAME", + "bucket name for Tencent COS, used for auto backup", "Please input your Tencent COS bucket name for backup"), + ConfigVar("CRUPEST_GITHUB_USERNAME", + "github username for fetching todos", "Please input your github username for fetching todos", "crupest"), + ConfigVar("CRUPEST_GITHUB_PROJECT_NUMBER", + "github project number for fetching todos", "Please input your github project number for fetching todos", "2"), + ConfigVar("CRUPEST_GITHUB_TOKEN", + "github token for fetching todos", "Please input your github token for fetching todos"), + ConfigVar("CRUPEST_GITHUB_TODO_COUNT", + "github todo count", "Please input your github todo count", 10), +] + +config_var_name_set = set([config_var.name for config_var in config_var_list]) + + +def check_config_var_set(needed_config_var_set: set): + more = [] + less = [] + for var_name in needed_config_var_set: + if var_name not in config_var_name_set: + more.append(var_name) + for var_name in config_var_name_set: + if var_name not in needed_config_var_set: + less.append(var_name) + return (True if len(more) == 0 else False, more, less) diff --git a/tool/modules/path.py b/tool/modules/path.py index 5d9aa3a..c4cf35b 100644 --- a/tool/modules/path.py +++ b/tool/modules/path.py @@ -1,3 +1,4 @@ +import os import os.path script_dir = os.path.relpath(os.path.dirname(__file__)) @@ -9,3 +10,8 @@ data_dir = os.path.join(project_dir, "data") tool_dir = os.path.join(project_dir, "tool") config_file_path = os.path.join(data_dir, "config") nginx_config_dir = os.path.join(project_dir, "nginx-config") +log_dir = os.path.join(project_dir, "log") + +def ensure_log_dir(): + if not os.path.exists(log_dir): + os.mkdir(log_dir) diff --git a/tool/test-crupest-api.py b/tool/test-crupest-api.py new file mode 100755 index 0000000..5cd461d --- /dev/null +++ b/tool/test-crupest-api.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 + +import subprocess +import os +import sys +from os.path import * +import signal +from modules.path import * +import time +from rich.console import Console +from urllib.request import urlopen +from http.client import * +import json + +console = console = Console() + +ensure_log_dir() + +dotnet_project = join(project_dir, "docker", "crupest-api", "CrupestApi") +dotnet_log_path = abspath(join(log_dir, "crupest-api-log")) +dotnet_config_path = abspath(join(project_dir, "crupest-api-config.json")) + +os.environ["CRUPEST_API_CONFIG_FILE"] = dotnet_log_path +os.environ["CRUPEST_API_LOG_FILE"] = dotnet_config_path + +popen = subprocess.Popen( + ["dotnet", "run", "--project", dotnet_project, "--launch-profile", "dev"] +) + +console.print("Sleep for 3s to wait for server startup.") +time.sleep(3) + + +def do_the_test(): + res: HTTPResponse = urlopen("http://localhost:5188/api/todos") + console.print(res) + body = res.read() + console.print(body) + + if res.status != 200: + raise Exception("Status code is not 200.") + result = json.load(body) + if not isinstance(result, list): + raise Exception("Result is not an array.") + if len(result) == 0: + raise Exception("Result is an empty array.") + if not isinstance(result[0], dict): + raise Exception("Result[0] is not an object.") + if not isinstance(result[0].get("title"), str): + raise Exception("Result[0].title is not a string.") + if not isinstance(result[0].get("status"), str): + raise Exception("Result[0].status is not a string.") + + +for i in range(0, 2): + console.print(f"Test begin with attempt {i + 1}", style="cyan") + try: + do_the_test() + console.print("Test passed.", style="green") + popen.send_signal(signal.SIGTERM) + popen.wait() + exit(0) + except Exception as e: + console.print(e) + console.print( + "Test failed. Try again after sleep for 1s.", style="red") + time.sleep(1) + +try: + console.print( + f"Test begin with attempt {i + 2}, also the final one.", style="cyan") + do_the_test() + console.print("Test passed.", style="green") + popen.send_signal(signal.SIGTERM) + popen.wait() + exit(0) +except Exception as e: + console.print(e) + console.print("Final test failed.", style="red") + exit(1) |