aboutsummaryrefslogtreecommitdiff
path: root/docker/auto-backup
diff options
context:
space:
mode:
Diffstat (limited to 'docker/auto-backup')
-rw-r--r--docker/auto-backup/.dockerignore2
-rw-r--r--docker/auto-backup/AutoBackup/.dockerignore2
-rw-r--r--docker/auto-backup/AutoBackup/.gitignore2
-rw-r--r--docker/auto-backup/AutoBackup/AutoBackup.csproj10
-rw-r--r--docker/auto-backup/AutoBackup/Program.cs121
-rw-r--r--docker/auto-backup/AutoBackup/TencentCloudCOS.cs211
-rw-r--r--docker/auto-backup/Dockerfile24
-rwxr-xr-xdocker/auto-backup/daemon.bash57
8 files changed, 429 insertions, 0 deletions
diff --git a/docker/auto-backup/.dockerignore b/docker/auto-backup/.dockerignore
new file mode 100644
index 0000000..7a09751
--- /dev/null
+++ b/docker/auto-backup/.dockerignore
@@ -0,0 +1,2 @@
+AutoBackup/bin
+AutoBackup/obj
diff --git a/docker/auto-backup/AutoBackup/.dockerignore b/docker/auto-backup/AutoBackup/.dockerignore
new file mode 100644
index 0000000..7de5508
--- /dev/null
+++ b/docker/auto-backup/AutoBackup/.dockerignore
@@ -0,0 +1,2 @@
+obj
+bin
diff --git a/docker/auto-backup/AutoBackup/.gitignore b/docker/auto-backup/AutoBackup/.gitignore
new file mode 100644
index 0000000..7de5508
--- /dev/null
+++ b/docker/auto-backup/AutoBackup/.gitignore
@@ -0,0 +1,2 @@
+obj
+bin
diff --git a/docker/auto-backup/AutoBackup/AutoBackup.csproj b/docker/auto-backup/AutoBackup/AutoBackup.csproj
new file mode 100644
index 0000000..694035b
--- /dev/null
+++ b/docker/auto-backup/AutoBackup/AutoBackup.csproj
@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <OutputType>Exe</OutputType>
+ <TargetFramework>net9.0</TargetFramework>
+ <ImplicitUsings>enable</ImplicitUsings>
+ <Nullable>enable</Nullable>
+ </PropertyGroup>
+
+</Project>
diff --git a/docker/auto-backup/AutoBackup/Program.cs b/docker/auto-backup/AutoBackup/Program.cs
new file mode 100644
index 0000000..c2e7a0d
--- /dev/null
+++ b/docker/auto-backup/AutoBackup/Program.cs
@@ -0,0 +1,121 @@
+// Read args to determine what file to upload
+
+const string DefaultUploadFilePath = "/tmp/data.tar.xz";
+string uploadFilePath = DefaultUploadFilePath;
+string? uploadDestinationPath = null;
+if (args.Length == 0)
+{
+ Console.WriteLine("You don't specify the file to upload, will upload /tmp/data.tar.xz by default.");
+ Console.WriteLine("You don't specify the destination to upload, will use timestamp with proper file extension.");
+}
+else if (args.Length == 1)
+{
+ if (args[0].Length == 0)
+ {
+ Console.Error.WriteLine("File to upload can't be empty string.");
+ Environment.Exit(2);
+ }
+ uploadFilePath = args[0];
+ Console.WriteLine("You don't specify the destination to upload, will use timestamp with proper file extension.");
+}
+else if (args.Length == 2)
+{
+ if (args[0].Length == 0)
+ {
+ Console.Error.WriteLine("File to upload can't be empty string.");
+ Environment.Exit(2);
+ }
+
+ if (args[1].Length == 0)
+ {
+ Console.Error.WriteLine("Destination to upload can't be empty string.");
+ Environment.Exit(2);
+ }
+
+ uploadFilePath = args[0];
+ uploadDestinationPath = args[1];
+}
+else
+{
+ // Write to stderr
+ Console.Error.WriteLine("You can only specify one optional file and one optional destination to upload.");
+ Environment.Exit(2);
+}
+
+// Check the upload exists
+if (!File.Exists(uploadFilePath))
+{
+ Console.Error.WriteLine($"The file {uploadFilePath} doesn't exist.");
+ Environment.Exit(3);
+}
+
+// Check the upload file is not a directory
+if (File.GetAttributes(uploadFilePath).HasFlag(FileAttributes.Directory))
+{
+ Console.Error.WriteLine($"The file {uploadFilePath} is a directory.");
+ Environment.Exit(4);
+}
+
+// Check the upload file is not bigger than 5G
+if (new FileInfo(uploadFilePath).Length > 5L * 1024L * 1024L * 1024L)
+{
+ Console.Error.WriteLine($"The file {uploadFilePath} is bigger than 5G, which is not support now.");
+ Environment.Exit(5);
+}
+
+// Get config from environment variables
+var configNameList = new List<string>{
+ "CRUPEST_AUTO_BACKUP_COS_SECRET_ID",
+ "CRUPEST_AUTO_BACKUP_COS_SECRET_KEY",
+ "CRUPEST_AUTO_BACKUP_COS_REGION",
+ "CRUPEST_AUTO_BACKUP_BUCKET_NAME"
+};
+
+var config = new Dictionary<string, string>();
+foreach (var configName in configNameList)
+{
+ var configValue = Environment.GetEnvironmentVariable(configName);
+ if (configValue is null)
+ {
+ Console.Error.WriteLine($"Environment variable {configName} is required.");
+ Environment.Exit(5);
+ }
+ config.Add(configName, configValue);
+}
+
+var region = config["CRUPEST_AUTO_BACKUP_COS_REGION"];
+var secretId = config["CRUPEST_AUTO_BACKUP_COS_SECRET_ID"];
+var secretKey = config["CRUPEST_AUTO_BACKUP_COS_SECRET_KEY"];
+var bucketName = config["CRUPEST_AUTO_BACKUP_BUCKET_NAME"];
+
+var credentials = new TencentCloudCOSHelper.Credentials(secretId, secretKey);
+
+if (uploadDestinationPath is null)
+{
+ var uploadFileName = Path.GetFileName(uploadFilePath);
+ var firstDotPosition = uploadFileName.IndexOf('.');
+ uploadDestinationPath = DateTime.Now.ToString("s");
+ if (firstDotPosition != -1)
+ {
+ uploadDestinationPath += uploadFileName.Substring(firstDotPosition + 1);
+ }
+}
+
+Console.WriteLine($"Upload file source: {uploadFilePath}");
+Console.WriteLine($"Upload COS region: {config["CRUPEST_AUTO_BACKUP_COS_REGION"]}");
+Console.WriteLine($"Upload bucket name: {config["CRUPEST_AUTO_BACKUP_BUCKET_NAME"]}");
+Console.WriteLine($"Upload file destination: {uploadDestinationPath}");
+
+await using var fileStream = new FileStream(uploadFilePath, FileMode.Open, FileAccess.Read);
+
+// 上传对象
+try
+{
+ await TencentCloudCOSHelper.PutObject(credentials, region, bucketName, uploadDestinationPath, fileStream);
+ Console.WriteLine("Upload completed!");
+}
+catch (Exception e)
+{
+ Console.Error.WriteLine("Exception: " + e);
+ Environment.Exit(6);
+}
diff --git a/docker/auto-backup/AutoBackup/TencentCloudCOS.cs b/docker/auto-backup/AutoBackup/TencentCloudCOS.cs
new file mode 100644
index 0000000..28d032c
--- /dev/null
+++ b/docker/auto-backup/AutoBackup/TencentCloudCOS.cs
@@ -0,0 +1,211 @@
+using System.Net;
+using System.Security.Cryptography;
+using System.Text;
+
+
+public static class TencentCloudCOSHelper
+{
+ public class Credentials
+ {
+ public Credentials(string secretId, string secretKey)
+ {
+ SecretId = secretId;
+ SecretKey = secretKey;
+ }
+
+ public string SecretId { get; }
+ public string SecretKey { get; }
+ }
+
+ public class RequestInfo
+ {
+ public RequestInfo(string method, string urlPathname, IEnumerable<KeyValuePair<string, string>> parameters, IEnumerable<KeyValuePair<string, string>> headers)
+ {
+ Method = method;
+ UrlPathname = urlPathname;
+ Parameters = new Dictionary<string, string>(parameters);
+ Headers = new Dictionary<string, string>(headers);
+ }
+
+ public string Method { get; }
+ public string UrlPathname { get; }
+ public IReadOnlyDictionary<string, string> Parameters { get; }
+ public IReadOnlyDictionary<string, string> Headers { get; }
+ }
+
+ public class TimeDuration
+ {
+ public TimeDuration(DateTimeOffset start, DateTimeOffset end)
+ {
+ if (start > end)
+ {
+ throw new ArgumentException("Start time must be earlier than end time.");
+ }
+
+ Start = start;
+ End = end;
+ }
+
+ public DateTimeOffset Start { get; }
+ public DateTimeOffset End { get; }
+ }
+
+ public static string GenerateSign(Credentials credentials, RequestInfo request, TimeDuration signValidTime)
+ {
+ List<(string key, string value)> Transform(IEnumerable<KeyValuePair<string, string>> raw)
+ {
+ if (raw == null)
+ return new List<(string key, string value)>();
+
+ var sorted = raw.Select(p => (key: WebUtility.UrlEncode(p.Key.ToLower()), value: WebUtility.UrlEncode(p.Value))).ToList();
+ sorted.Sort((left, right) => string.CompareOrdinal(left.key, right.key));
+ return sorted;
+ }
+
+ var transformedParameters = Transform(request.Parameters);
+ var transformedHeaders = Transform(request.Headers);
+
+
+ const string signAlgorithm = "sha1";
+
+ static string ByteArrayToString(byte[] bytes)
+ {
+ return BitConverter.ToString(bytes).Replace("-", "").ToLower();
+ }
+
+ var keyTime = $"{signValidTime.Start.ToUnixTimeSeconds().ToString()};{signValidTime.End.ToUnixTimeSeconds().ToString()}";
+ using HMACSHA1 hmac = new HMACSHA1(Encoding.ASCII.GetBytes(credentials.SecretKey));
+ var signKey = ByteArrayToString(hmac.ComputeHash(Encoding.UTF8.GetBytes(keyTime)));
+
+ static string Join(IEnumerable<(string key, string value)> raw)
+ {
+ return string.Join('&', raw.Select(p => string.Concat(p.key, "=", p.value)));
+ }
+
+ var httpParameters = Join(transformedParameters);
+ var urlParamList = string.Join(';', transformedParameters.Select(p => p.key));
+ var httpHeaders = Join(transformedHeaders);
+ var headerList = string.Join(';', transformedHeaders.Select(h => h.key));
+
+ var httpString = new StringBuilder()
+ .Append(request.Method.ToLower()).Append('\n')
+ .Append(request.UrlPathname).Append('\n')
+ .Append(httpParameters).Append('\n')
+ .Append(httpHeaders).Append('\n')
+ .ToString();
+
+ using var sha1 = SHA1.Create();
+ string Sha1(string data)
+ {
+ var result = sha1.ComputeHash(Encoding.UTF8.GetBytes(data));
+ return ByteArrayToString(result);
+ }
+
+ var stringToSign = new StringBuilder()
+ .Append(signAlgorithm).Append('\n')
+ .Append(keyTime).Append('\n')
+ .Append(Sha1(httpString)).Append('\n')
+ .ToString();
+
+ hmac.Key = Encoding.UTF8.GetBytes(signKey);
+ var signature = ByteArrayToString(hmac.ComputeHash(
+ Encoding.UTF8.GetBytes(stringToSign)));
+
+
+ List<(string, string)> result = new List<(string, string)>();
+ result.Add(("q-sign-algorithm", signAlgorithm));
+ result.Add(("q-ak", credentials.SecretId));
+ result.Add(("q-sign-time", keyTime));
+ result.Add(("q-key-time", keyTime));
+ result.Add(("q-header-list", headerList));
+ result.Add(("q-url-param-list", urlParamList));
+ result.Add(("q-signature", signature));
+ return Join(result);
+ }
+
+ private static string GetHost(string bucket, string region)
+ {
+ return $"{bucket}.cos.{region}.myqcloud.com";
+ }
+
+ public static async Task<bool> IsObjectExists(Credentials credentials, string region, string bucket, string key)
+ {
+ var host = GetHost(bucket, region);
+ var encodedKey = WebUtility.UrlEncode(key);
+
+ using var request = new HttpRequestMessage();
+ request.Method = HttpMethod.Head;
+ request.RequestUri = new Uri($"https://{host}/{encodedKey}");
+ request.Headers.Host = host;
+ request.Headers.Date = DateTimeOffset.Now;
+ request.Headers.TryAddWithoutValidation("Authorization", GenerateSign(credentials, new RequestInfo(
+ "head", "/" + key, new Dictionary<string, string>(),
+ new Dictionary<string, string>
+ {
+ ["Host"] = host
+ }
+ ), new TimeDuration(DateTimeOffset.Now, DateTimeOffset.Now.AddMinutes(5))));
+
+ using var client = new HttpClient();
+ using var response = await client.SendAsync(request);
+
+ if (response.IsSuccessStatusCode)
+ return true;
+ if (response.StatusCode == HttpStatusCode.NotFound)
+ return false;
+
+ throw new Exception($"Unknown response code. {response.ToString()}");
+ }
+
+ public static async Task PutObject(Credentials credentials, string region, string bucket, string key, Stream dataStream)
+ {
+ if (!dataStream.CanSeek)
+ {
+ throw new ArgumentException("Data stream must be seekable.");
+ }
+
+ if (dataStream.Seek(0, SeekOrigin.End) > 5L * 1024L * 1024L * 1024L)
+ {
+ throw new ArgumentException("Data stream must be smaller than 5GB.");
+ }
+
+ dataStream.Seek(0, SeekOrigin.Begin);
+
+ var host = GetHost(bucket, region);
+ var encodedKey = WebUtility.UrlEncode(key);
+ using var md5Handler = MD5.Create();
+ var md5 = Convert.ToBase64String(await md5Handler.ComputeHashAsync(dataStream));
+
+ dataStream.Seek(0, SeekOrigin.Begin);
+
+ const string kContentMD5HeaderName = "Content-MD5";
+
+ using var httpRequest = new HttpRequestMessage()
+ {
+ Method = HttpMethod.Put,
+ RequestUri = new Uri($"https://{host}/{encodedKey}")
+ };
+ httpRequest.Headers.Host = host;
+ httpRequest.Headers.Date = DateTimeOffset.Now;
+
+ using var httpContent = new StreamContent(dataStream);
+ httpContent.Headers.Add(kContentMD5HeaderName, md5);
+ httpRequest.Content = httpContent;
+
+ var signedHeaders = new Dictionary<string, string>
+ {
+ ["Host"] = host,
+ [kContentMD5HeaderName] = md5
+ };
+
+ httpRequest.Headers.TryAddWithoutValidation("Authorization", GenerateSign(credentials, new RequestInfo(
+ "put", "/" + key, new Dictionary<string, string>(), signedHeaders
+ ), new TimeDuration(DateTimeOffset.Now, DateTimeOffset.Now.AddMinutes(10))));
+
+ using var client = new HttpClient();
+ using var response = await client.SendAsync(httpRequest);
+
+ if (!response.IsSuccessStatusCode)
+ throw new Exception($"Not success status code: {response.StatusCode}\n{await response.Content.ReadAsStringAsync()}");
+ }
+} \ No newline at end of file
diff --git a/docker/auto-backup/Dockerfile b/docker/auto-backup/Dockerfile
new file mode 100644
index 0000000..c7ff4fc
--- /dev/null
+++ b/docker/auto-backup/Dockerfile
@@ -0,0 +1,24 @@
+FROM mcr.microsoft.com/dotnet/sdk:9.0-alpine AS build
+COPY AutoBackup /AutoBackup
+WORKDIR /AutoBackup
+RUN dotnet publish AutoBackup.csproj --configuration Release --output ./publish/ -r linux-x64 --self-contained false
+
+FROM mcr.microsoft.com/dotnet/runtime:9.0-alpine
+RUN apk add --no-cache tini coreutils bash tar xz
+ARG CRUPEST_AUTO_BACKUP_INIT_DELAY=0
+ARG CRUPEST_AUTO_BACKUP_INTERVAL=1d
+ARG CRUPEST_AUTO_BACKUP_COS_SECRET_ID
+ARG CRUPEST_AUTO_BACKUP_COS_SECRET_KEY
+ARG CRUPEST_AUTO_BACKUP_COS_REGION
+ARG CRUPEST_AUTO_BACKUP_BUCKET_NAME
+ENV CRUPEST_AUTO_BACKUP_INIT_DELAY=${CRUPEST_AUTO_BACKUP_INIT_DELAY}
+ENV CRUPEST_AUTO_BACKUP_INTERVAL=${CRUPEST_AUTO_BACKUP_INTERVAL}
+ENV CRUPEST_AUTO_BACKUP_COS_SECRET_ID=${CRUPEST_AUTO_BACKUP_COS_SECRET_ID}
+ENV CRUPEST_AUTO_BACKUP_COS_SECRET_KEY=${CRUPEST_AUTO_BACKUP_COS_SECRET_KEY}
+ENV CRUPEST_AUTO_BACKUP_COS_REGION=${CRUPEST_AUTO_BACKUP_COS_REGION}
+ENV CRUPEST_AUTO_BACKUP_BUCKET_NAME=${CRUPEST_AUTO_BACKUP_BUCKET_NAME}
+VOLUME [ "/data" ]
+COPY daemon.bash /daemon.bash
+COPY --from=build /AutoBackup/publish /AutoBackup
+ENTRYPOINT ["tini", "--"]
+CMD [ "/daemon.bash" ]
diff --git a/docker/auto-backup/daemon.bash b/docker/auto-backup/daemon.bash
new file mode 100755
index 0000000..a4dd5dc
--- /dev/null
+++ b/docker/auto-backup/daemon.bash
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Check I'm root.
+if [[ $EUID -ne 0 ]]; then
+ echo "This script must be run as root" 1>&2
+ exit 1
+fi
+
+
+# Check xz, tar and coscmd
+xz --version
+tar --version
+
+function backup {
+ # Output "Begin backup..." in yellow and restore default
+ echo -e "\e[0;103m\e[K\e[1mBegin backup..." "\e[0m"
+
+ # Get current time and convert it to YYYY-MM-DDTHH:MM:SSZ
+ current_time=$(date +%Y-%m-%dT%H:%M:%SZ)
+ echo "Current time: $current_time"
+
+ echo "Create tar.xz for data..."
+
+ # tar and xz /data to tmp
+ tar -cJf /tmp/data.tar.xz -C / data
+
+ # Output /tmp/data.tar.xz size
+ du -h /tmp/data.tar.xz | cut -f1 | xargs echo "Size of data.tar.xz:"
+
+ destination="${current_time}.tar.xz"
+
+ # upload to remote
+ dotnet /AutoBackup/AutoBackup.dll /tmp/data.tar.xz "$destination"
+
+ echo "Remove tmp file..."
+ # remove tmp
+ rm /tmp/data.tar.xz
+
+ echo "$destination" >> /data/backup.log
+
+ # echo "Backup finished!" in green and restore default
+ echo -e "\e[0;102m\e[K\e[1mFinish backup!\e[0m"
+}
+
+echo "Initial delay: $CRUPEST_AUTO_BACKUP_INIT_DELAY"
+sleep "$CRUPEST_AUTO_BACKUP_INIT_DELAY"
+
+# forever loop
+while true; do
+ backup
+
+ # sleep for CRUPEST_AUTO_BACKUP_INTERVAL
+ echo "Sleep for $CRUPEST_AUTO_BACKUP_INTERVAL for next backup..."
+ sleep "$CRUPEST_AUTO_BACKUP_INTERVAL"
+done
generated by cgit v1.2.3 (git 2.39.1) at 2025-04-18 16:48:01 +0000