refactor(secret): add cn geosite direct. refactor geodata manager.

7 files changed, 287 insertions, 64 deletions

diff --git a/tools/Crupest.V2ray/Crupest.V2ray/GeoDataDownloader.cs b/tools/Crupest.V2ray/Crupest.V2ray/GeoDataDownloader.cs
deleted file mode 100644
index d3efc1f..0000000
--- a/tools/Crupest.V2ray/Crupest.V2ray/GeoDataDownloader.cs
+++ /dev/null
@@ -1,42 +0,0 @@
-namespace Crupest.V2ray;
-
-public class GeoDataDownloader
-{
-    public record GithubReleaseAsset(string ResourceName, string User, string Repo, string AssetName, string Output);
-
-    public GeoDataDownloader()
-    {
-        Resources = new()
-        {
-            new("geosite", "v2fly", "domain-list-community", "dlc.dat", "geosite.dat"),
-            new("geoip", "v2fly", "geoip", "geoip.dat", "geoip.dat"),
-            new("geosite", "v2fly", "geoip", "geoip-only-cn-private.dat", "geoip-only-cn-private.dat")
-        };
-    }
-
-    public List<GithubReleaseAsset> Resources { get; set; }
-
-    public static string GetReleaseFileUrl(string user, string repo, string assetName)
-    {
-        return $"https://github.com/{user}/{repo}/releases/latest/download/{assetName}";
-    }
-
-    public static void GithubDownload(HttpClient httpClient, string user, string repo, string assetName, string outputPath)
-    {
-        using var responseStream = httpClient.GetStreamAsync(GetReleaseFileUrl(user, repo, assetName)).Result;
-        using var outputFileStream = File.OpenWrite(outputPath);
-        responseStream.CopyTo(outputFileStream);
-    }
-
-    public void Download(string outputDir)
-    {
-        using var httpClient = new HttpClient();
-
-        foreach (var resource in Resources)
-        {
-            Console.WriteLine($"Downloading {resource.ResourceName}...");
-            GithubDownload(httpClient, resource.User, resource.Repo, resource.AssetName, Path.Combine(outputDir, resource.Output));
-            Console.WriteLine($"Downloaded {resource.ResourceName}!");
-        }
-    }
-}
diff --git a/tools/Crupest.V2ray/Crupest.V2ray/GeoDataManager.cs b/tools/Crupest.V2ray/Crupest.V2ray/GeoDataManager.cs
new file mode 100644
index 0000000..75f071b
--- /dev/null
+++ b/tools/Crupest.V2ray/Crupest.V2ray/GeoDataManager.cs
@@ -0,0 +1,209 @@
+using System.IO.Compression;
+
+namespace Crupest.V2ray;
+
+public interface IGeoSiteEntry
+{
+    bool IsInclude { get; }
+    string Value { get; }
+}
+
+public record GeoSiteIncludeEntry(string Value, string ContainingSite) : IGeoSiteEntry
+{
+    public bool IsInclude => true;
+}
+
+public record GeoSiteRuleEntry(V2rayHostMatcherKind Kind, string Value, List<string> Attributes, string ContainingSite) : IGeoSiteEntry
+{
+    public bool IsInclude => false;
+}
+
+public record GeoSite(string Name, List<IGeoSiteEntry> Entries)
+{
+    public static GeoSite Parse(string name, string str)
+    {
+        List<IGeoSiteEntry> entries = [];
+        var listConfig = new ListConfig(str);
+        foreach (var item in listConfig.Config)
+        {
+            var (value, line) = item;
+
+            if (value.StartsWith("include:"))
+            {
+                var include = value["include:".Length..].Trim();
+                if (include.Length == 0 || include.Contains(' '))
+                {
+                    throw new FormatException($"Invalid geo site rule in line {line}. Invalid include value.");
+                }
+                entries.Add(new GeoSiteIncludeEntry(include, name));
+                continue;
+            }
+
+            var segments = value.Split(':', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
+            if (segments.Length > 2)
+            {
+                throw new FormatException($"Invalid geo site rule in line {line}. More than one ':'.");
+            }
+
+            V2rayHostMatcherKind kind;
+            if (segments.Length == 2)
+            {
+                kind = segments[0] switch
+                {
+                    "domain" => kind = V2rayHostMatcherKind.DomainSuffix,
+                    "full" => kind = V2rayHostMatcherKind.DomainFull,
+                    "keyword" => kind = V2rayHostMatcherKind.DomainKeyword,
+                    "regexp" => kind = V2rayHostMatcherKind.DomainRegex,
+                    _ => throw new FormatException($"Invalid geo site rule in line {line}. Unknown matcher.")
+                };
+            }
+            else
+            {
+                kind = V2rayHostMatcherKind.DomainSuffix;
+            }
+
+            var domainSegments = segments[^1].Split('@', StringSplitOptions.TrimEntries);
+            var domain = domainSegments[0];
+            if (Uri.CheckHostName(domain) != UriHostNameType.Dns)
+            {
+                throw new FormatException($"Invalid geo site rule in line {line}. Invalid domain.");
+            }
+
+            List<string> attributes = [];
+            foreach (var s in domainSegments)
+            {
+                if (s.Length == 0)
+                {
+                    throw new FormatException($"Invalid geo site rule in line {line}. Empty attribute value.");
+                }
+                attributes.Add(s);
+            }
+
+            entries.Add(new GeoSiteRuleEntry(kind, domain, attributes, name));
+        }
+        return new GeoSite(name, entries);
+    }
+}
+
+public class GeoSiteDataParser(string directory)
+{
+    private static List<GeoSite> Parse(string directory)
+    {
+        var sites = new List<GeoSite>();
+        foreach (var file in Directory.GetFileSystemEntries(directory))
+        {
+            var path = Path.Combine(directory, file);
+            var content = File.ReadAllText(path);
+            sites.Add(GeoSite.Parse(file, content));
+        }
+        return sites;
+    }
+
+    public string DataDirectory { get; } = directory;
+
+    public List<GeoSite> Sites { get; } = Parse(directory);
+}
+
+public class GeoDataManager
+{
+    public const string GeoSiteFileName = "geosite.dat";
+    public const string GeoIpFileName = "geoip.dat";
+    public const string GeoIpCnFileName = "geoip-only-cn-private.dat";
+    public const string V2rayGithubOrganization = "v2fly";
+    public const string V2rayGeoSiteGithubRepository = "domain-list-community";
+    public const string V2rayGeoIpGithubRepository = "geoip";
+    public const string V2rayGeoSiteCnGithubReleaseFilename = "dlc.dat";
+    public const string V2rayGeoIpGithubReleaseFilename = "geoip.dat";
+    public const string V2rayGeoIpCnGithubReleaseFilename = "geoip-only-cn-private.dat";
+
+    public static GeoDataManager Instance { get; } = new GeoDataManager();
+
+    public record GeoDataAsset(string Name, string FileName, string GithubUser, string GithubRepo, string GithubReleaseFileName);
+
+    public GeoDataManager()
+    {
+        Assets =
+        [
+            new("geosite", GeoSiteFileName, V2rayGithubOrganization, V2rayGeoSiteGithubRepository, V2rayGeoSiteGithubRepository),
+            new("geoip", GeoIpFileName, V2rayGithubOrganization, V2rayGeoIpGithubRepository, V2rayGeoIpGithubReleaseFilename),
+            new("geoip-cn", GeoIpCnFileName, V2rayGithubOrganization, V2rayGeoIpGithubRepository, V2rayGeoIpCnGithubReleaseFilename),
+        ];
+    }
+
+    public List<GeoDataAsset> Assets { get; set; }
+
+    private static string GetReleaseFileUrl(string user, string repo, string fileName)
+    {
+        return $"https://github.com/{user}/{repo}/releases/latest/download/{fileName}";
+    }
+
+    private static void GithubDownloadRelease(HttpClient httpClient, string user, string repo, string fileName, string outputPath)
+    {
+        using var responseStream = httpClient.GetStreamAsync(GetReleaseFileUrl(user, repo, fileName)).Result;
+        using var outputFileStream = File.OpenWrite(outputPath);
+        responseStream.CopyTo(outputFileStream);
+    }
+
+    public bool HasAllAssets(string directory, out List<string> missing)
+    {
+        missing = [];
+        foreach (var asset in Assets)
+        {
+            var assetPath = Path.Combine(directory, asset.FileName);
+            if (!File.Exists(assetPath))
+            {
+                missing.Add(asset.Name);
+            }
+        }
+        return missing.Count == 0;
+    }
+
+    public void Download(string outputDir, bool silent)
+    {
+        using var httpClient = new HttpClient();
+
+        foreach (var asset in Assets)
+        {
+            if (!silent)
+            {
+                Console.WriteLine($"Downloading {asset.Name}...");
+            }
+            GithubDownloadRelease(httpClient, asset.GithubUser, asset.GithubRepo, asset.GithubReleaseFileName, Path.Combine(outputDir, asset.FileName));
+            if (!silent)
+            {
+                Console.WriteLine($"Downloaded {asset.Name}!");
+            }
+        }
+    }
+
+    private static string GetGithubRepositoryArchiveUrl(string user, string repo)
+    {
+        return $"https://github.com/{user}/{repo}/archive/refs/heads/master.zip";
+    }
+
+    private static void GithubDownloadRepository(HttpClient httpClient, string user, string repo, string outputPath)
+    {
+        using var responseStream = httpClient.GetStreamAsync(GetGithubRepositoryArchiveUrl(user, repo)).Result;
+        using var outputFileStream = File.OpenWrite(outputPath);
+        responseStream.CopyTo(outputFileStream);
+    }
+
+    private static void Unzip(string zipPath, string outputPath)
+    {
+        using var zip = ZipFile.OpenRead(zipPath) ?? throw new Exception($"Failed to open zip file {zipPath}");
+        zip.ExtractToDirectory(outputPath);
+    }
+
+    private string DownloadAndExtractGeoSiteRepository(bool silent)
+    {
+        const string zipFileName = "v2ray-geosite-master.zip";
+        using var httpClient = new HttpClient();
+        var tempDirectory = Directory.CreateTempSubdirectory(Program.Name);
+        var archivePath = Path.Combine(tempDirectory.FullName, zipFileName);
+        var extractPath = Path.Combine(tempDirectory.FullName, "repo");
+        GithubDownloadRepository(httpClient, V2rayGithubOrganization, V2rayGeoSiteGithubRepository, archivePath);
+        Directory.CreateDirectory(extractPath);
+        Unzip(archivePath, extractPath);
+        return Path.Join(extractPath, "domain-list-community-master");
+    }
+}
diff --git a/tools/Crupest.V2ray/Crupest.V2ray/Program.cs b/tools/Crupest.V2ray/Crupest.V2ray/Program.cs
index e06a92d..a40aa21 100644
--- a/tools/Crupest.V2ray/Crupest.V2ray/Program.cs
+++ b/tools/Crupest.V2ray/Crupest.V2ray/Program.cs
@@ -4,6 +4,8 @@ namespace Crupest.V2ray;
 
 public static class Program
 {
+    public static string Name { get; } = typeof(Program).Namespace ?? throw new Exception("Can't get the name of Crupest.V2ray.");
+
     public static string CrupestV2rayDirectory { get; } =
         Environment.GetEnvironmentVariable("CRUPEST_V2RAY_DIR") ??
         Path.GetFullPath(Path.GetDirectoryName(
@@ -13,10 +15,31 @@ public static class Program
 
     public static void RunV2rayAndWatchConfigChange()
     {
-        var v2rayPath = V2rayController.FindExecutable(CrupestV2rayDirectory) ??
+        var v2rayPath = V2rayController.FindExecutable(CrupestV2rayDirectory, out var isLocal) ??
             throw new Exception("Can't find v2ray executable either in Crupest.V2ray directory or in PATH.");
 
-        var v2rayController = new V2rayController(v2rayPath, Path.Combine(CrupestV2rayDirectory, ConfigOutputFileName), CrupestV2rayDirectory);
+        string? assetsPath;
+        if (isLocal)
+        {
+            assetsPath = CrupestV2rayDirectory;
+            var assetsComplete = GeoDataManager.Instance.HasAllAssets(CrupestV2rayDirectory, out var missing);
+            if (!assetsComplete)
+            {
+                throw new Exception($"Missing assets: {string.Join(", ", missing)} in {CrupestV2rayDirectory}. This v2ray is local. So only use assets in Crupest.V2ray directory.");
+            }
+        }
+        else
+        {
+            assetsPath = CrupestV2rayDirectory;
+            var assetsComplete = GeoDataManager.Instance.HasAllAssets(CrupestV2rayDirectory, out var missing);
+            if (!assetsComplete)
+            {
+                Console.WriteLine($"Missing assets: {string.Join(", ", missing)} in {CrupestV2rayDirectory}. This v2ray is global. So fallback to its own assets.");
+                assetsPath = null;
+            }
+        }
+
+        var v2rayController = new V2rayController(v2rayPath, Path.Combine(CrupestV2rayDirectory, ConfigOutputFileName), assetsPath);
         var configFileWatcher = new FileWatcher(CrupestV2rayDirectory, V2rayConfig.ConfigFileNames);
 
         V2rayConfig.FromDirectoryAndWriteToFile(CrupestV2rayDirectory, Path.Join(CrupestV2rayDirectory, ConfigOutputFileName));
@@ -42,8 +65,7 @@ public static class Program
             var verb = args[0].ToLower();
             if (verb == "download-geodata" || verb == "dg")
             {
-                var geoDataDownloader = new GeoDataDownloader();
-                geoDataDownloader.Download(CrupestV2rayDirectory);
+                GeoDataManager.Instance.Download(CrupestV2rayDirectory, false);
                 return;
             }
             else if (verb == "generate" || verb == "g")
diff --git a/tools/Crupest.V2ray/Crupest.V2ray/V2rayConfig.cs b/tools/Crupest.V2ray/Crupest.V2ray/V2rayConfig.cs
index c246bb5..3758f47 100644
--- a/tools/Crupest.V2ray/Crupest.V2ray/V2rayConfig.cs
+++ b/tools/Crupest.V2ray/Crupest.V2ray/V2rayConfig.cs
@@ -42,6 +42,14 @@ public class V2rayConfig(Template template, List<V2rayProxy> proxies, V2rayRouti
     private const string RoutingAnchor = "ROUTING_ANCHOR";
     private const string HostsAnchor = "HOSTS_ANCHOR";
 
+    public const string AddCnAttributeToGeositeEnvironmentVariable = "CRUPEST_V@RAY_GEOSITE_USE_CN";
+
+    private static bool UseCnGeoSite => Environment.GetEnvironmentVariable(AddCnAttributeToGeositeEnvironmentVariable) switch
+    {
+        "0" or "false" or "off" or "disable" => false,
+        _ => true
+    };
+
     public Template Template { get; set; } = template;
     public List<V2rayProxy> Proxies { get; set; } = proxies;
     public V2rayRouting Routing { get; set; } = router;
@@ -118,7 +126,7 @@ public class V2rayConfig(Template template, List<V2rayProxy> proxies, V2rayRouti
             file = vmessPath;
             var vmess = V2rayVmessProxy.CreateFromConfigString(vmessString, "proxy");
             file = proxyPath;
-            var routing = V2rayRouting.CreateFromConfigString(routingString, "proxy");
+            var routing = V2rayRouting.CreateFromConfigString(routingString, "proxy", UseCnGeoSite);
             file = hostsPath ?? "";
             var hosts = hostsString is not null ? V2rayHosts.CreateFromConfigString(hostsString) : null;
             return new V2rayConfig(template, [vmess], routing, hosts);
diff --git a/tools/Crupest.V2ray/Crupest.V2ray/V2rayController.cs b/tools/Crupest.V2ray/Crupest.V2ray/V2rayController.cs
index ab1614a..4656216 100644
--- a/tools/Crupest.V2ray/Crupest.V2ray/V2rayController.cs
+++ b/tools/Crupest.V2ray/Crupest.V2ray/V2rayController.cs
@@ -2,12 +2,13 @@ using System.Diagnostics;
 
 namespace Crupest.V2ray;
 
-public class V2rayController(string executablePath, string configPath, string assetPath)
+public class V2rayController(string executablePath, string configPath, string? assetPath)
 {
     public const string V2rayAssetEnvironmentVariableName = "v2ray.location.asset";
 
-    public static string? FindExecutable(string contentDir, string? executableName = null)
+    public static string? FindExecutable(string contentDir, out bool isLocal, string? executableName = null)
     {
+        isLocal = false;
         executableName ??= "v2ray";
 
         if (OperatingSystem.IsWindows())
@@ -18,6 +19,7 @@ public class V2rayController(string executablePath, string configPath, string as
         var localV2rayPath = Path.Combine(contentDir, executableName);
         if (File.Exists(localV2rayPath))
         {
+            isLocal = true;
             return localV2rayPath;
         }
 
@@ -39,7 +41,7 @@ public class V2rayController(string executablePath, string configPath, string as
 
     public string ExecutablePath { get; } = executablePath;
     public string ConfigPath { get; } = configPath;
-    public string AssetPath { get; } = assetPath;
+    public string? AssetPath { get; } = assetPath;
     public Process? CurrentProcess { get; private set; }
 
     private Process CreateProcess()
@@ -53,7 +55,10 @@ public class V2rayController(string executablePath, string configPath, string as
         startInfo.ArgumentList.Add("run");
         startInfo.ArgumentList.Add("-c");
         startInfo.ArgumentList.Add(ConfigPath);
-        startInfo.EnvironmentVariables[V2rayAssetEnvironmentVariableName] = AssetPath;
+        if (AssetPath is not null)
+        {
+            startInfo.EnvironmentVariables[V2rayAssetEnvironmentVariableName] = AssetPath;
+        }
 
         process.StartInfo = startInfo;
         process.OutputDataReceived += (_, args) =>
diff --git a/tools/Crupest.V2ray/Crupest.V2ray/V2rayRouting.cs b/tools/Crupest.V2ray/Crupest.V2ray/V2rayRouting.cs
index 528b49e..080a7b4 100644
--- a/tools/Crupest.V2ray/Crupest.V2ray/V2rayRouting.cs
+++ b/tools/Crupest.V2ray/Crupest.V2ray/V2rayRouting.cs
@@ -17,9 +17,12 @@ public record V2rayRoutingRule(V2rayHostMatcherKind MatcherKind, string MatcherS
     public static Dictionary<string, List<V2rayRoutingRule>> GroupByOutboundTag(List<V2rayRoutingRule> rules)
         => rules.GroupBy(r => r.OutboundTag).Select(g => (g.Key, g.ToList())).ToDictionary();
 
-    public static Dictionary<V2rayHostMatcherKind, List<V2rayRoutingRule>> GroupByMatcherByKind(List<V2rayRoutingRule> rules)
+    public static Dictionary<V2rayHostMatcherKind, List<V2rayRoutingRule>> GroupByMatcherKind(List<V2rayRoutingRule> rules)
         => rules.GroupBy(r => r.MatcherKind).Select(g => (g.Key, g.ToList())).ToDictionary();
 
+    public static List<List<V2rayRoutingRule>> GroupByOutboundTagAndMatcherKind(List<V2rayRoutingRule> rules)
+        => GroupByOutboundTag(rules).Values.SelectMany((groupByTag) => GroupByMatcherKind(groupByTag).Values).ToList();
+
     public static V2rayV4ConfigJsonObjects.RoutingRule ListToJsonObject(List<V2rayRoutingRule> rules)
     {
         if (rules.Count == 0)
@@ -43,30 +46,48 @@ public record V2rayRoutingRule(V2rayHostMatcherKind MatcherKind, string MatcherS
         );
     }
 
+    public V2rayRoutingRule CloneGeositeWithCnAttribute(string outboundTag)
+    {
+        if (MatcherKind is not V2rayHostMatcherKind.GeoSite)
+        {
+            throw new ArgumentException("Matcher kind must be GeoSite.");
+        }
+
+        return new V2rayRoutingRule(V2rayHostMatcherKind.GeoSite, $"{MatcherString}@cn", outboundTag);
+    }
+
     public V2rayV4ConfigJsonObjects.RoutingRule ToJsonObjectV4() => ListToJsonObject([this]);
 
     object IV2rayV4ConfigObject.ToJsonObjectV4() => ToJsonObjectV4();
 }
 
-public record V2rayRouting(List<V2rayRoutingRule> Rules, string DomainStrategy = "IpOnDemand") : IV2rayV4ConfigObject
+public record V2rayRouting(List<V2rayRoutingRule> Rules, bool DirectGeositeCn = true, string DomainStrategy = "IpOnDemand") : IV2rayV4ConfigObject
 {
-    public V2rayV4ConfigJsonObjects.Routing ToJsonObjectV4()
+    public List<V2rayRoutingRule> CreateGeositeCnDirectRules()
     {
-        var ruleJsonObjects = new List<object>();
+        return Rules.Where(r => r.MatcherKind is V2rayHostMatcherKind.GeoSite)
+            .Select(r => r.CloneGeositeWithCnAttribute("direct")).ToList();
+    }
+
+    public V2rayV4ConfigJsonObjects.Routing ToJsonObjectV4(bool directGeositeCn = true)
+    {
+        List<V2rayV4ConfigJsonObjects.RoutingRule> ruleJsonObjects = [];
+
+        if (directGeositeCn)
+        {
+            ruleJsonObjects.Add(V2rayRoutingRule.ListToJsonObject(CreateGeositeCnDirectRules()));
+        }
 
-        var rules = V2rayRoutingRule.GroupByOutboundTag(Rules).ToList().SelectMany((groupByTag) =>
-            V2rayRoutingRule.GroupByMatcherByKind(groupByTag.Value).ToList().Select((groupByMatcher) =>
-                V2rayRoutingRule.ListToJsonObject(groupByMatcher.Value))
-        ).ToList();
+        ruleJsonObjects.AddRange(V2rayRoutingRule.GroupByOutboundTagAndMatcherKind(Rules).Select(V2rayRoutingRule.ListToJsonObject));
 
-        return new V2rayV4ConfigJsonObjects.Routing(rules);
+        return new V2rayV4ConfigJsonObjects.Routing(ruleJsonObjects);
     }
 
     object IV2rayV4ConfigObject.ToJsonObjectV4() => ToJsonObjectV4();
 
-    public static V2rayRouting CreateFromConfigString(string configString, string outboundTag)
+    public static V2rayRouting CreateFromConfigString(string configString, string outboundTag, bool directGeositeCn = true)
     {
         var matcherConfig = new V2rayHostMatcherConfig(configString, [.. Enum.GetValues<V2rayHostMatcherKind>()], maxComponentCount: 0);
-        return new V2rayRouting(matcherConfig.Items.Select(i => new V2rayRoutingRule(i.Kind, i.Matcher, outboundTag)).ToList());
+        return new V2rayRouting(matcherConfig.Items.Select(i => new V2rayRoutingRule(i.Kind, i.Matcher, outboundTag)).ToList(), directGeositeCn);
     }
 }
diff --git a/tools/Crupest.V2ray/Crupest.V2ray/config.json.template b/tools/Crupest.V2ray/Crupest.V2ray/config.json.template
index 424e996..686006c 100644
--- a/tools/Crupest.V2ray/Crupest.V2ray/config.json.template
+++ b/tools/Crupest.V2ray/Crupest.V2ray/config.json.template
@@ -4,7 +4,7 @@
   },
   "inbounds": [
     {
-      "port": 2081,
+      "port": 3081,
       "listen": "127.0.0.1",
       "tag": "socks-inbound",
       "protocol": "socks",
@@ -13,7 +13,7 @@
       }
     },
     {
-      "port": 2080,
+      "port": 3080,
       "listen": "127.0.0.1",
       "tag": "http-inbound",
       "protocol": "http",