Skip to content
This repository was archived by the owner on Jul 3, 2020. It is now read-only.

Commit ac52931

Browse files
OpportunityLiuOpportunityLiu
committed
fix tag checker
1 parent 6936179 commit ac52931

File tree

4 files changed

+122
-84
lines changed

4 files changed

+122
-84
lines changed

‎EhDbReleaseBuilder/GitHubApiClient.cs‎

Lines changed: 2 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ public class HeadData
4848
public JRaw Data { get; set; }
4949
}
5050

51+
public Task CheckAsync(Namespace checkTags) => TagChecker.CheckAsync(_Database, checkTags);
52+
5153
private class FullUploadData : UploadData
5254
{
5355
public HeadData Head { get; set; }
@@ -59,78 +61,6 @@ public void Normalize()
5961
_Database.Save();
6062
}
6163

62-
private void _LogFailed(RecordDictionary db, string key, Record value, Namespace newNs, string newKey)
63-
{
64-
db.AddOrReplace(key, new Record(value.Name.Raw, value.Intro.Raw, value.Links.Raw
65-
+ $"\nNow should be {newNs}:{newKey}"));
66-
Console.WriteLine(" -> Failed");
67-
}
68-
69-
public async Task CheckAsync(Namespace checkTags)
70-
{
71-
try
72-
{
73-
// skip rows & reclass
74-
foreach (var db in _Database.Values.Where(v => v.Namespace > Namespace.Reclass && checkTags.HasFlag(v.Namespace)))
75-
{
76-
var ns = db.Namespace;
77-
Console.WriteLine($"Checking namespace {ns} with {db.RawData.Count} items");
78-
for (var i = 0; i < db.RawData.Count; i++)
79-
{
80-
var data = db.RawData[i];
81-
Console.Write($"[{i,6}/{db.RawData.Count}] {data.Key}");
82-
if (string.IsNullOrWhiteSpace(data.Key))
83-
{
84-
Console.WriteLine(" -> Skipped empty tag");
85-
continue;
86-
}
87-
var (newNs, newKey) = await TagChecker.CheckAsync(ns, data.Key);
88-
if (newKey is null)
89-
{
90-
db.Remove(data.Key, false);
91-
Console.WriteLine(" -> Delete");
92-
}
93-
else if (newNs != ns)
94-
{
95-
Console.Write($" -> Move to {newNs}:{newKey}");
96-
try
97-
{
98-
_Database[newNs].Add(newKey, data.Value);
99-
db.Remove(data.Key, true);
100-
Console.WriteLine(" -> Succeed");
101-
}
102-
catch
103-
{
104-
_LogFailed(db, data.Key, data.Value, newNs, newKey);
105-
}
106-
}
107-
else if (newKey != data.Key)
108-
{
109-
Console.Write($" -> Rename to {newKey}");
110-
try
111-
{
112-
db.Rename(data.Key, newKey);
113-
Console.WriteLine(" -> Succeed");
114-
}
115-
catch
116-
{
117-
_LogFailed(db, data.Key, data.Value, newNs, newKey);
118-
}
119-
}
120-
else
121-
{
122-
Console.WriteLine(" -> Valid");
123-
}
124-
}
125-
_Database.Save();
126-
}
127-
}
128-
finally
129-
{
130-
_Database.Save();
131-
}
132-
}
133-
13464
public void Publish()
13565
{
13666
foreach (var item in Directory.CreateDirectory(_Target).GetFiles())

‎EhDbReleaseBuilder/Program.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ public static async Task Main(string source, string target, Namespace checkTags
2121
Console.WriteLine($@"EhDbReleaseBuilder started.
2222
Source: {source}
2323
Target: {target}
24-
Check tags: {checkTags}
24+
Check tags: {(checkTags > Namespace.Reclass ? checkTags.ToString() : "<Disabled>")}
2525
");
2626
var client = new GitHubApiClient(source, target);
2727
if (checkTags > Namespace.Reclass)

‎EhDbReleaseBuilder/Properties/launchSettings.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"profiles": {
33
"EhDbReleaseBuilder": {
44
"commandName": "Project",
5-
"commandLineArgs": "--source C:/Users/lzy/Documents/Source/EhTagTranslation/EhTagConnector/EHTT-DB/ --target C:/Users/lzy/Documents/Source/EhTagTranslation/EhTagConnector/EHTT-DB/publish --check-tags Misc,Male,Female,Parody,Character,Group,Artist"
5+
"commandLineArgs": "--source C:/Users/lzy/Documents/Source/EhTagTranslation/EhTagConnector/EHTT-DB/ --target C:/Users/lzy/Documents/Source/EhTagTranslation/EhTagConnector/EHTT-DB/publish --check-tags Parody,Character,Group,Artist"
66
}
77
}
88
}

‎EhDbReleaseBuilder/TagChecker.cs‎

Lines changed: 118 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,35 @@ private static Tag _FindCache(Namespace ns, string raw)
5151
return tag;
5252
}
5353

54-
public static async Task<(Namespace, string)> CheckAsync(Namespace ns, string raw)
54+
private static async Task<TagSuggest> _PostApiAsync(Namespace? ns, string raw)
55+
{
56+
var text = raw.Length > 50 ? raw.Substring(0, 50) : raw;
57+
if (ns != null)
58+
text = ns + ":" + raw;
59+
var response = await _HttpClient.PostAsync("https://api.e-hentai.org/api.php", new StringContent(JsonConvert.SerializeObject(new
60+
{
61+
method = "tagsuggest",
62+
text,
63+
})));
64+
var resultStr = await response.Content.ReadAsStringAsync();
65+
if (resultStr.Contains("{\"tags\":[]}"))
66+
{
67+
if (raw.Contains('.'))
68+
{
69+
return await _PostApiAsync(ns, raw.Substring(0, raw.IndexOf('.') - 1));
70+
}
71+
else
72+
{
73+
return null;
74+
}
75+
}
76+
77+
var result = JsonConvert.DeserializeObject<TagSuggest>(resultStr);
78+
_FillCache(result);
79+
return result;
80+
}
81+
82+
public static async Task<(Namespace, string)> _CheckTagAsync(Namespace ns, string raw)
5583
{
5684
if (raw.Length <= 2)
5785
return (ns, raw);
@@ -60,22 +88,26 @@ private static Tag _FindCache(Namespace ns, string raw)
6088

6189
if (match is null)
6290
{
63-
var response = await _HttpClient.PostAsync("https://api.e-hentai.org/api.php", new StringContent(JsonConvert.SerializeObject(new
64-
{
65-
method = "tagsuggest",
66-
text = $"{(raw.Length > 50 ? raw.Substring(0, 50) : raw)}",
67-
})));
68-
var resultStr = await response.Content.ReadAsStringAsync();
69-
if (!resultStr.Contains("{\"tags\":[]}"))
91+
var result = await _PostApiAsync(null, raw);
92+
if (result != null)
7093
{
71-
var result = JsonConvert.DeserializeObject<TagSuggest>(resultStr);
72-
_FillCache(result);
7394
// check exact match first
7495
match = result.tags.Values.FirstOrDefault(tag => tag.ns == ns && tag.tn == raw)
7596
// find from misc ns and master in this ns
7697
?? result.tags.Values.FirstOrDefault(tag => tag.ns == Namespace.Misc && tag.tn == raw && tag.mns == ns);
7798
}
99+
}
78100

101+
if (match is null)
102+
{
103+
var result = await _PostApiAsync(ns, raw);
104+
if (result != null)
105+
{
106+
// check exact match first
107+
match = result.tags.Values.FirstOrDefault(tag => tag.ns == ns && tag.tn == raw)
108+
// find from misc ns and master in this ns
109+
?? result.tags.Values.FirstOrDefault(tag => tag.ns == Namespace.Misc && tag.tn == raw && tag.mns == ns);
110+
}
79111
}
80112

81113
if (match is null)
@@ -84,5 +116,81 @@ private static Tag _FindCache(Namespace ns, string raw)
84116
return (ns, raw);
85117
return (match.mns ?? Namespace.Misc, match.mtn);
86118
}
119+
120+
private static void _LogFailed(RecordDictionary db, string key, Record value, Namespace newNs, string newKey)
121+
{
122+
db.AddOrReplace(key, new Record(value.Name.Raw, value.Intro.Raw, value.Links.Raw
123+
+ $"\nNow should be {newNs}:{newKey}"));
124+
Console.WriteLine(" -> Failed");
125+
}
126+
private static async Task _CheckNsAsync(Database database, Namespace ns)
127+
{
128+
var db = database[ns];
129+
Console.WriteLine($"Checking namespace {ns} with {db.RawData.Count} lines");
130+
for (var i = 0; i < db.RawData.Count; i++)
131+
{
132+
var data = db.RawData[i];
133+
Console.Write($" [{i,4}/{db.RawData.Count}] {data.Key}");
134+
if (string.IsNullOrWhiteSpace(data.Key))
135+
{
136+
Console.WriteLine(" -> Skipped empty tag");
137+
continue;
138+
}
139+
var (newNs, newKey) = await _CheckTagAsync(ns, data.Key);
140+
if (newKey is null)
141+
{
142+
db.Remove(data.Key, false);
143+
Console.WriteLine(" -> Delete");
144+
}
145+
else if (newNs != ns)
146+
{
147+
Console.Write($" -> Move to {newNs}:{newKey}");
148+
try
149+
{
150+
database[newNs].Add(newKey, data.Value);
151+
db.Remove(data.Key, true);
152+
Console.WriteLine(" -> Succeed");
153+
}
154+
catch
155+
{
156+
_LogFailed(db, data.Key, data.Value, newNs, newKey);
157+
}
158+
}
159+
else if (newKey != data.Key)
160+
{
161+
Console.Write($" -> Rename to {newKey}");
162+
try
163+
{
164+
db.Rename(data.Key, newKey);
165+
Console.WriteLine(" -> Succeed");
166+
}
167+
catch
168+
{
169+
_LogFailed(db, data.Key, data.Value, newNs, newKey);
170+
}
171+
}
172+
else
173+
{
174+
Console.WriteLine(" -> Valid");
175+
}
176+
}
177+
}
178+
179+
public static async Task CheckAsync(Database database, Namespace checkTags)
180+
{
181+
try
182+
{
183+
// skip rows & reclass
184+
foreach (var ns in database.Keys.Where(v => v > Namespace.Reclass && checkTags.HasFlag(v)))
185+
{
186+
await _CheckNsAsync(database, ns);
187+
database.Save();
188+
}
189+
}
190+
finally
191+
{
192+
database.Save();
193+
}
194+
}
87195
}
88196
}

0 commit comments

Comments
 (0)