From 3d7307adb372b648675f637e71c5f8c9d6e90620 Mon Sep 17 00:00:00 2001 From: crupest Date: Thu, 24 Nov 2022 10:07:35 +0800 Subject: Use cos sdk and my own cli. --- docker/auto-backup/.dockerignore | 2 + docker/auto-backup/AutoBackup/.gitignore | 2 + docker/auto-backup/AutoBackup/AutoBackup.csproj | 14 +++ docker/auto-backup/AutoBackup/Program.cs | 141 ++++++++++++++++++++++++ docker/auto-backup/Dockerfile | 19 ++-- docker/auto-backup/daemon.bash | 16 +-- 6 files changed, 173 insertions(+), 21 deletions(-) create mode 100644 docker/auto-backup/.dockerignore create mode 100644 docker/auto-backup/AutoBackup/.gitignore create mode 100644 docker/auto-backup/AutoBackup/AutoBackup.csproj create mode 100644 docker/auto-backup/AutoBackup/Program.cs (limited to 'docker/auto-backup') diff --git a/docker/auto-backup/.dockerignore b/docker/auto-backup/.dockerignore new file mode 100644 index 0000000..7a09751 --- /dev/null +++ b/docker/auto-backup/.dockerignore @@ -0,0 +1,2 @@ +AutoBackup/bin +AutoBackup/obj diff --git a/docker/auto-backup/AutoBackup/.gitignore b/docker/auto-backup/AutoBackup/.gitignore new file mode 100644 index 0000000..7de5508 --- /dev/null +++ b/docker/auto-backup/AutoBackup/.gitignore @@ -0,0 +1,2 @@ +obj +bin diff --git a/docker/auto-backup/AutoBackup/AutoBackup.csproj b/docker/auto-backup/AutoBackup/AutoBackup.csproj new file mode 100644 index 0000000..24000b2 --- /dev/null +++ b/docker/auto-backup/AutoBackup/AutoBackup.csproj @@ -0,0 +1,14 @@ + + + + Exe + net7.0 + enable + enable + + + + + + + diff --git a/docker/auto-backup/AutoBackup/Program.cs b/docker/auto-backup/AutoBackup/Program.cs new file mode 100644 index 0000000..0b5845a --- /dev/null +++ b/docker/auto-backup/AutoBackup/Program.cs @@ -0,0 +1,141 @@ +using COSXML; +using COSXML.Auth; +using COSXML.Transfer; + +// Check I'm root +if (Environment.UserName != "root") +{ + Console.WriteLine("You must run this program as root"); + Environment.Exit(1); +} + +// Read args to determine what file to upload + +const string DefaultUploadFilePath = "/tmp/data.tar.xz"; +string uploadFilePath = DefaultUploadFilePath; +string? uploadDestinationPath = null; +if (args.Length == 0) +{ + Console.WriteLine("You don't specify the file to upload, will upload /tmp/data.tar.xz by default."); + Console.WriteLine("You don't specify the destination to upload, will use timestamp with proper file extension."); +} +else if (args.Length == 1) +{ + if (args[0].Length == 0) + { + Console.Error.WriteLine("File to upload can't be empty string."); + Environment.Exit(2); + } + uploadFilePath = args[0]; + Console.WriteLine("You don't specify the destination to upload, will use timestamp with proper file extension."); +} +else if (args.Length == 2) +{ + if (args[0].Length == 0) + { + Console.Error.WriteLine("File to upload can't be empty string."); + Environment.Exit(2); + } + + if (args[1].Length == 0) + { + Console.Error.WriteLine("Destination to upload can't be empty string."); + Environment.Exit(2); + } + + uploadFilePath = args[0]; + uploadDestinationPath = args[1]; +} +else +{ + // Write to stderr + Console.Error.WriteLine("You can only specify one optional file and one optional destination to upload."); + Environment.Exit(2); +} + +// Check the upload exists +if (!File.Exists(uploadFilePath)) +{ + Console.Error.WriteLine($"The file {uploadFilePath} doesn't exist."); + Environment.Exit(3); +} + +// Check the upload file is not a directory +if (File.GetAttributes(uploadFilePath).HasFlag(FileAttributes.Directory)) +{ + Console.Error.WriteLine($"The file {uploadFilePath} is a directory."); + Environment.Exit(4); +} + +// Get config from environment variables +var configNameList = new List{ + "CRUPEST_AUTO_BACKUP_COS_SECRET_ID", + "CRUPEST_AUTO_BACKUP_COS_SECRET_KEY", + "CRUPEST_AUTO_BACKUP_COS_REGION", + "CRUPEST_AUTO_BACKUP_BUCKET_NAME" +}; + +var config = new Dictionary(); +foreach (var configName in configNameList) +{ + var configValue = Environment.GetEnvironmentVariable(configName); + if (configValue is null) + { + Console.Error.WriteLine($"Environment variable {configName} is required."); + Environment.Exit(5); + } +} + +var cosConfig = new CosXmlConfig.Builder() + .IsHttps(true) + .SetRegion(config["CRUPEST_AUTO_BACKUP_COS_REGION"]) + .Build(); + +QCloudCredentialProvider cosCredentialProvider = + new DefaultQCloudCredentialProvider( + config["CRUPEST_AUTO_BACKUP_COS_SECRET_ID"], + config["CRUPEST_AUTO_BACKUP_COS_SECRET_KEY"], + 60 + ); + +CosXml cosXml = new CosXmlServer(cosConfig, cosCredentialProvider); + +TransferConfig transferConfig = new TransferConfig(); + +TransferManager transferManager = new TransferManager(cosXml, transferConfig); + +if (uploadDestinationPath is null) +{ + var uploadFileName = Path.GetFileName(uploadFilePath); + var firstDotPosition = uploadFileName.IndexOf('.'); + uploadDestinationPath = DateTime.Now.ToString("s"); + if (firstDotPosition != -1) + { + uploadDestinationPath += uploadFileName.Substring(firstDotPosition + 1); + } +} + +Console.WriteLine($"Upload file source: {uploadFilePath}"); +Console.WriteLine($"Upload COS region: {config["CRUPEST_AUTO_BACKUP_COS_REGION"]}"); +Console.WriteLine($"Upload bucket name: {config["CRUPEST_AUTO_BACKUP_BUCKET_NAME"]}"); +Console.WriteLine($"Upload file destination: {uploadDestinationPath}"); + +// 上传对象 +COSXMLUploadTask uploadTask = new COSXMLUploadTask(config["CRUPEST_AUTO_BACKUP_BUCKET_NAME"], uploadDestinationPath); +uploadTask.SetSrcPath(uploadFilePath); + +uploadTask.progressCallback = delegate (long completed, long total) +{ + Console.WriteLine(String.Format("progress = {0:##.##}%", completed * 100.0 / total)); +}; + +try +{ + COSXMLUploadTask.UploadTaskResult result = await transferManager.UploadAsync(uploadTask); + Console.WriteLine("Upload completed!"); +} +catch (Exception e) +{ + Console.Error.WriteLine("CosException: " + e); + Environment.Exit(6); +} diff --git a/docker/auto-backup/Dockerfile b/docker/auto-backup/Dockerfile index 51cbfa9..7adf5ef 100644 --- a/docker/auto-backup/Dockerfile +++ b/docker/auto-backup/Dockerfile @@ -1,5 +1,10 @@ -FROM alpine:latest -RUN apk add --no-cache build-base coreutils bash tar xz python3 +FROM mcr.microsoft.com/dotnet/sdk:7.0-alpine AS build +COPY AutoBackup /AutoBackup +WORKDIR /AutoBackup +RUN dotnet publish AutoBackup.csproj --configuration Release --output ./publish/ -r linux-x64 --self-contained false + +FROM mcr.microsoft.com/dotnet/aspnet:7.0-alpine +RUN apk add --no-cache coreutils bash tar xz ARG CRUPEST_AUTO_BACKUP_INIT_DELAY=0 ARG CRUPEST_AUTO_BACKUP_INTERVAL=1d ARG CRUPEST_AUTO_BACKUP_COS_SECRET_ID @@ -12,14 +17,8 @@ ENV CRUPEST_AUTO_BACKUP_COS_SECRET_ID=${CRUPEST_AUTO_BACKUP_COS_SECRET_ID} ENV CRUPEST_AUTO_BACKUP_COS_SECRET_KEY=${CRUPEST_AUTO_BACKUP_COS_SECRET_KEY} ENV CRUPEST_AUTO_BACKUP_COS_REGION=${CRUPEST_AUTO_BACKUP_COS_REGION} ENV CRUPEST_AUTO_BACKUP_BUCKET_NAME=${CRUPEST_AUTO_BACKUP_BUCKET_NAME} -# install pip and coscmd -RUN python3 -m ensurepip && \ - rm -r /usr/lib/python*/ensurepip && \ - pip3 install --upgrade pip setuptools && \ - pip3 install coscmd && \ - rm -r /root/.cache && \ - coscmd --version -COPY daemon.bash /daemon.bash VOLUME [ "/data" ] +COPY daemon.bash /daemon.bash +COPY --from=build /AutoBackup/publish /AutoBackup STOPSIGNAL SIGINT ENTRYPOINT [ "/daemon.bash" ] diff --git a/docker/auto-backup/daemon.bash b/docker/auto-backup/daemon.bash index f425d10..845ad38 100755 --- a/docker/auto-backup/daemon.bash +++ b/docker/auto-backup/daemon.bash @@ -12,18 +12,15 @@ fi # Check xz, tar and coscmd xz --version tar --version -coscmd --version - -# do not echo next command -coscmd config -a "${CRUPEST_AUTO_BACKUP_COS_SECRET_ID}" -s "${CRUPEST_AUTO_BACKUP_COS_SECRET_KEY}" -b "${CRUPEST_AUTO_BACKUP_BUCKET_NAME}" -r "${CRUPEST_AUTO_BACKUP_COS_REGION}" function backup { # Output "Begin backup..." in yellow and restore default echo -e "\e[0;103m\e[K\e[1mBegin backup..." "\e[0m" - # Get current time and convert it to YYYY-MM-DDTHH.MM.SS - current_time=$(date +%Y-%m-%dT%H.%M.%S) + # Get current time and convert it to YYYY-MM-DDTHH:MM:SS + current_time=$(date +%Y-%m-%dT%H:%M:%S) echo "Current time: $current_time" + echo "Create tar.xz for data..." # tar and xz /data to tmp @@ -32,13 +29,10 @@ function backup { # Output /tmp/data.tar.xz size du -h /tmp/data.tar.xz | cut -f1 | xargs echo "Size of data.tar.xz:" - destination="$current_time.tar.xz" - echo "Use coscli to upload data to $destination ..." - echo "Bucket name: ${CRUPEST_AUTO_BACKUP_BUCKET_NAME}" - echo "Bucket region: ${CRUPEST_AUTO_BACKUP_COS_REGION}" + destination="${current_time}.tar.xz" # upload to remote - coscmd upload /tmp/data.tar.xz "$destination" + dotnet /AutoBackup/AutoBackup.dll /tmp/data.tar.xz "$destination" echo "Remove tmp file..." # remove tmp -- cgit v1.2.3