From 5edf355f564de2c8b6f9ab28bd6f77654029f0f3 Mon Sep 17 00:00:00 2001 From: Yuqian Yang Date: Wed, 19 Feb 2025 02:01:34 +0800 Subject: feat(auto-backup): use coscli. --- docker/auto-backup/AutoBackup/.dockerignore | 2 - docker/auto-backup/AutoBackup/.gitignore | 2 - docker/auto-backup/AutoBackup/AutoBackup.csproj | 10 -- docker/auto-backup/AutoBackup/Program.cs | 121 ------------- docker/auto-backup/AutoBackup/TencentCloudCOS.cs | 211 ----------------------- 5 files changed, 346 deletions(-) delete mode 100644 docker/auto-backup/AutoBackup/.dockerignore delete mode 100644 docker/auto-backup/AutoBackup/.gitignore delete mode 100644 docker/auto-backup/AutoBackup/AutoBackup.csproj delete mode 100644 docker/auto-backup/AutoBackup/Program.cs delete mode 100644 docker/auto-backup/AutoBackup/TencentCloudCOS.cs (limited to 'docker/auto-backup/AutoBackup') diff --git a/docker/auto-backup/AutoBackup/.dockerignore b/docker/auto-backup/AutoBackup/.dockerignore deleted file mode 100644 index 7de5508..0000000 --- a/docker/auto-backup/AutoBackup/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -obj -bin diff --git a/docker/auto-backup/AutoBackup/.gitignore b/docker/auto-backup/AutoBackup/.gitignore deleted file mode 100644 index 7de5508..0000000 --- a/docker/auto-backup/AutoBackup/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -obj -bin diff --git a/docker/auto-backup/AutoBackup/AutoBackup.csproj b/docker/auto-backup/AutoBackup/AutoBackup.csproj deleted file mode 100644 index 694035b..0000000 --- a/docker/auto-backup/AutoBackup/AutoBackup.csproj +++ /dev/null @@ -1,10 +0,0 @@ - - - - Exe - net9.0 - enable - enable - - - diff --git a/docker/auto-backup/AutoBackup/Program.cs b/docker/auto-backup/AutoBackup/Program.cs deleted file mode 100644 index c2e7a0d..0000000 --- a/docker/auto-backup/AutoBackup/Program.cs +++ /dev/null @@ -1,121 +0,0 @@ -// Read args to determine what file to upload - -const string DefaultUploadFilePath = "/tmp/data.tar.xz"; -string uploadFilePath = DefaultUploadFilePath; -string? uploadDestinationPath = null; -if (args.Length == 0) -{ - Console.WriteLine("You don't specify the file to upload, will upload /tmp/data.tar.xz by default."); - Console.WriteLine("You don't specify the destination to upload, will use timestamp with proper file extension."); -} -else if (args.Length == 1) -{ - if (args[0].Length == 0) - { - Console.Error.WriteLine("File to upload can't be empty string."); - Environment.Exit(2); - } - uploadFilePath = args[0]; - Console.WriteLine("You don't specify the destination to upload, will use timestamp with proper file extension."); -} -else if (args.Length == 2) -{ - if (args[0].Length == 0) - { - Console.Error.WriteLine("File to upload can't be empty string."); - Environment.Exit(2); - } - - if (args[1].Length == 0) - { - Console.Error.WriteLine("Destination to upload can't be empty string."); - Environment.Exit(2); - } - - uploadFilePath = args[0]; - uploadDestinationPath = args[1]; -} -else -{ - // Write to stderr - Console.Error.WriteLine("You can only specify one optional file and one optional destination to upload."); - Environment.Exit(2); -} - -// Check the upload exists -if (!File.Exists(uploadFilePath)) -{ - Console.Error.WriteLine($"The file {uploadFilePath} doesn't exist."); - Environment.Exit(3); -} - -// Check the upload file is not a directory -if (File.GetAttributes(uploadFilePath).HasFlag(FileAttributes.Directory)) -{ - Console.Error.WriteLine($"The file {uploadFilePath} is a directory."); - Environment.Exit(4); -} - -// Check the upload file is not bigger than 5G -if (new FileInfo(uploadFilePath).Length > 5L * 1024L * 1024L * 1024L) -{ - Console.Error.WriteLine($"The file {uploadFilePath} is bigger than 5G, which is not support now."); - Environment.Exit(5); -} - -// Get config from environment variables -var configNameList = new List{ - "CRUPEST_AUTO_BACKUP_COS_SECRET_ID", - "CRUPEST_AUTO_BACKUP_COS_SECRET_KEY", - "CRUPEST_AUTO_BACKUP_COS_REGION", - "CRUPEST_AUTO_BACKUP_BUCKET_NAME" -}; - -var config = new Dictionary(); -foreach (var configName in configNameList) -{ - var configValue = Environment.GetEnvironmentVariable(configName); - if (configValue is null) - { - Console.Error.WriteLine($"Environment variable {configName} is required."); - Environment.Exit(5); - } - config.Add(configName, configValue); -} - -var region = config["CRUPEST_AUTO_BACKUP_COS_REGION"]; -var secretId = config["CRUPEST_AUTO_BACKUP_COS_SECRET_ID"]; -var secretKey = config["CRUPEST_AUTO_BACKUP_COS_SECRET_KEY"]; -var bucketName = config["CRUPEST_AUTO_BACKUP_BUCKET_NAME"]; - -var credentials = new TencentCloudCOSHelper.Credentials(secretId, secretKey); - -if (uploadDestinationPath is null) -{ - var uploadFileName = Path.GetFileName(uploadFilePath); - var firstDotPosition = uploadFileName.IndexOf('.'); - uploadDestinationPath = DateTime.Now.ToString("s"); - if (firstDotPosition != -1) - { - uploadDestinationPath += uploadFileName.Substring(firstDotPosition + 1); - } -} - -Console.WriteLine($"Upload file source: {uploadFilePath}"); -Console.WriteLine($"Upload COS region: {config["CRUPEST_AUTO_BACKUP_COS_REGION"]}"); -Console.WriteLine($"Upload bucket name: {config["CRUPEST_AUTO_BACKUP_BUCKET_NAME"]}"); -Console.WriteLine($"Upload file destination: {uploadDestinationPath}"); - -await using var fileStream = new FileStream(uploadFilePath, FileMode.Open, FileAccess.Read); - -// 上传对象 -try -{ - await TencentCloudCOSHelper.PutObject(credentials, region, bucketName, uploadDestinationPath, fileStream); - Console.WriteLine("Upload completed!"); -} -catch (Exception e) -{ - Console.Error.WriteLine("Exception: " + e); - Environment.Exit(6); -} diff --git a/docker/auto-backup/AutoBackup/TencentCloudCOS.cs b/docker/auto-backup/AutoBackup/TencentCloudCOS.cs deleted file mode 100644 index 28d032c..0000000 --- a/docker/auto-backup/AutoBackup/TencentCloudCOS.cs +++ /dev/null @@ -1,211 +0,0 @@ -using System.Net; -using System.Security.Cryptography; -using System.Text; - - -public static class TencentCloudCOSHelper -{ - public class Credentials - { - public Credentials(string secretId, string secretKey) - { - SecretId = secretId; - SecretKey = secretKey; - } - - public string SecretId { get; } - public string SecretKey { get; } - } - - public class RequestInfo - { - public RequestInfo(string method, string urlPathname, IEnumerable> parameters, IEnumerable> headers) - { - Method = method; - UrlPathname = urlPathname; - Parameters = new Dictionary(parameters); - Headers = new Dictionary(headers); - } - - public string Method { get; } - public string UrlPathname { get; } - public IReadOnlyDictionary Parameters { get; } - public IReadOnlyDictionary Headers { get; } - } - - public class TimeDuration - { - public TimeDuration(DateTimeOffset start, DateTimeOffset end) - { - if (start > end) - { - throw new ArgumentException("Start time must be earlier than end time."); - } - - Start = start; - End = end; - } - - public DateTimeOffset Start { get; } - public DateTimeOffset End { get; } - } - - public static string GenerateSign(Credentials credentials, RequestInfo request, TimeDuration signValidTime) - { - List<(string key, string value)> Transform(IEnumerable> raw) - { - if (raw == null) - return new List<(string key, string value)>(); - - var sorted = raw.Select(p => (key: WebUtility.UrlEncode(p.Key.ToLower()), value: WebUtility.UrlEncode(p.Value))).ToList(); - sorted.Sort((left, right) => string.CompareOrdinal(left.key, right.key)); - return sorted; - } - - var transformedParameters = Transform(request.Parameters); - var transformedHeaders = Transform(request.Headers); - - - const string signAlgorithm = "sha1"; - - static string ByteArrayToString(byte[] bytes) - { - return BitConverter.ToString(bytes).Replace("-", "").ToLower(); - } - - var keyTime = $"{signValidTime.Start.ToUnixTimeSeconds().ToString()};{signValidTime.End.ToUnixTimeSeconds().ToString()}"; - using HMACSHA1 hmac = new HMACSHA1(Encoding.ASCII.GetBytes(credentials.SecretKey)); - var signKey = ByteArrayToString(hmac.ComputeHash(Encoding.UTF8.GetBytes(keyTime))); - - static string Join(IEnumerable<(string key, string value)> raw) - { - return string.Join('&', raw.Select(p => string.Concat(p.key, "=", p.value))); - } - - var httpParameters = Join(transformedParameters); - var urlParamList = string.Join(';', transformedParameters.Select(p => p.key)); - var httpHeaders = Join(transformedHeaders); - var headerList = string.Join(';', transformedHeaders.Select(h => h.key)); - - var httpString = new StringBuilder() - .Append(request.Method.ToLower()).Append('\n') - .Append(request.UrlPathname).Append('\n') - .Append(httpParameters).Append('\n') - .Append(httpHeaders).Append('\n') - .ToString(); - - using var sha1 = SHA1.Create(); - string Sha1(string data) - { - var result = sha1.ComputeHash(Encoding.UTF8.GetBytes(data)); - return ByteArrayToString(result); - } - - var stringToSign = new StringBuilder() - .Append(signAlgorithm).Append('\n') - .Append(keyTime).Append('\n') - .Append(Sha1(httpString)).Append('\n') - .ToString(); - - hmac.Key = Encoding.UTF8.GetBytes(signKey); - var signature = ByteArrayToString(hmac.ComputeHash( - Encoding.UTF8.GetBytes(stringToSign))); - - - List<(string, string)> result = new List<(string, string)>(); - result.Add(("q-sign-algorithm", signAlgorithm)); - result.Add(("q-ak", credentials.SecretId)); - result.Add(("q-sign-time", keyTime)); - result.Add(("q-key-time", keyTime)); - result.Add(("q-header-list", headerList)); - result.Add(("q-url-param-list", urlParamList)); - result.Add(("q-signature", signature)); - return Join(result); - } - - private static string GetHost(string bucket, string region) - { - return $"{bucket}.cos.{region}.myqcloud.com"; - } - - public static async Task IsObjectExists(Credentials credentials, string region, string bucket, string key) - { - var host = GetHost(bucket, region); - var encodedKey = WebUtility.UrlEncode(key); - - using var request = new HttpRequestMessage(); - request.Method = HttpMethod.Head; - request.RequestUri = new Uri($"https://{host}/{encodedKey}"); - request.Headers.Host = host; - request.Headers.Date = DateTimeOffset.Now; - request.Headers.TryAddWithoutValidation("Authorization", GenerateSign(credentials, new RequestInfo( - "head", "/" + key, new Dictionary(), - new Dictionary - { - ["Host"] = host - } - ), new TimeDuration(DateTimeOffset.Now, DateTimeOffset.Now.AddMinutes(5)))); - - using var client = new HttpClient(); - using var response = await client.SendAsync(request); - - if (response.IsSuccessStatusCode) - return true; - if (response.StatusCode == HttpStatusCode.NotFound) - return false; - - throw new Exception($"Unknown response code. {response.ToString()}"); - } - - public static async Task PutObject(Credentials credentials, string region, string bucket, string key, Stream dataStream) - { - if (!dataStream.CanSeek) - { - throw new ArgumentException("Data stream must be seekable."); - } - - if (dataStream.Seek(0, SeekOrigin.End) > 5L * 1024L * 1024L * 1024L) - { - throw new ArgumentException("Data stream must be smaller than 5GB."); - } - - dataStream.Seek(0, SeekOrigin.Begin); - - var host = GetHost(bucket, region); - var encodedKey = WebUtility.UrlEncode(key); - using var md5Handler = MD5.Create(); - var md5 = Convert.ToBase64String(await md5Handler.ComputeHashAsync(dataStream)); - - dataStream.Seek(0, SeekOrigin.Begin); - - const string kContentMD5HeaderName = "Content-MD5"; - - using var httpRequest = new HttpRequestMessage() - { - Method = HttpMethod.Put, - RequestUri = new Uri($"https://{host}/{encodedKey}") - }; - httpRequest.Headers.Host = host; - httpRequest.Headers.Date = DateTimeOffset.Now; - - using var httpContent = new StreamContent(dataStream); - httpContent.Headers.Add(kContentMD5HeaderName, md5); - httpRequest.Content = httpContent; - - var signedHeaders = new Dictionary - { - ["Host"] = host, - [kContentMD5HeaderName] = md5 - }; - - httpRequest.Headers.TryAddWithoutValidation("Authorization", GenerateSign(credentials, new RequestInfo( - "put", "/" + key, new Dictionary(), signedHeaders - ), new TimeDuration(DateTimeOffset.Now, DateTimeOffset.Now.AddMinutes(10)))); - - using var client = new HttpClient(); - using var response = await client.SendAsync(httpRequest); - - if (!response.IsSuccessStatusCode) - throw new Exception($"Not success status code: {response.StatusCode}\n{await response.Content.ReadAsStringAsync()}"); - } -} \ No newline at end of file -- cgit v1.2.3