|
|
1
|
+"""This script generates a HTML file from the results of ftbench"""
|
|
|
2
|
+import os
|
|
|
3
|
+import re
|
|
|
4
|
+import sys
|
|
|
5
|
+
|
|
|
6
|
+GITLAB_URL = "https://gitlab.freedesktop.org/freetype/freetype/-/commit/"
|
|
|
7
|
+CSS_STYLE = """
|
|
|
8
|
+ <style>
|
|
|
9
|
+ table {
|
|
|
10
|
+ table-layout: fixed;
|
|
|
11
|
+ }
|
|
|
12
|
+ th, td {
|
|
|
13
|
+ padding: 3px;
|
|
|
14
|
+ text-align: center;
|
|
|
15
|
+ }
|
|
|
16
|
+ th {
|
|
|
17
|
+ background-color: #ccc;
|
|
|
18
|
+ color: black;
|
|
|
19
|
+ }
|
|
|
20
|
+ .warning{
|
|
|
21
|
+ color: red;
|
|
|
22
|
+ }
|
|
|
23
|
+ .col1 {
|
|
|
24
|
+ background-color: #eee;
|
|
|
25
|
+ }
|
|
|
26
|
+
|
|
|
27
|
+
|
|
|
28
|
+ .highlight {
|
|
|
29
|
+ background-color: #0a0;
|
|
|
30
|
+ }
|
|
|
31
|
+ </style>
|
|
|
32
|
+"""
|
|
|
33
|
+OBJ_DIR = sys.argv[1]
|
|
|
34
|
+BASELINE_DIR = os.path.join(OBJ_DIR,"baseline")
|
|
|
35
|
+BENCHMARK_DIR = os.path.join(OBJ_DIR,"benchmark")
|
|
|
36
|
+BENCHMARK_HTML = os.path.join(OBJ_DIR,"benchmark.html")
|
|
|
37
|
+
|
|
|
38
|
+FONT_COUNT = 5
|
|
|
39
|
+
|
|
|
40
|
+WARNING_SAME_COMMIT = "Warning: Baseline and Benchmark have the same commit ID!"
|
|
|
41
|
+INFO_1 = "* Average time for single iteration. Smaller values are better."
|
|
|
42
|
+INFO_2 = "* If a value in the 'Iterations' column is given as '*x* | *y*', values *x* and *y* give the number of iterations in the baseline and the benchmark test, respectively."
|
|
|
43
|
+
|
|
|
44
|
+
|
|
|
45
|
+
|
|
|
46
|
+
|
|
|
47
|
+def main():
|
|
|
48
|
+ """Entry point for theq script"""
|
|
|
49
|
+ with open(BENCHMARK_HTML, "w") as html_file:
|
|
|
50
|
+ write_to_html(html_file, "<html>\n<head>\n")
|
|
|
51
|
+ write_to_html(html_file, CSS_STYLE)
|
|
|
52
|
+ write_to_html(html_file, "</head>\n<body>\n")
|
|
|
53
|
+ write_to_html(html_file, "<h1>Freetype Benchmark Results</h1>\n")
|
|
|
54
|
+
|
|
|
55
|
+ baseline_info = parse_info_file(os.path.join(BASELINE_DIR, "info.txt"))
|
|
|
56
|
+ benchmark_info = parse_info_file(os.path.join(BENCHMARK_DIR, "info.txt"))
|
|
|
57
|
+
|
|
|
58
|
+ if baseline_info[1].strip() == benchmark_info[1].strip():
|
|
|
59
|
+ write_to_html(
|
|
|
60
|
+ html_file,
|
|
|
61
|
+ f'<h2 class="warning">{WARNING_SAME_COMMIT}</h2>\n',
|
|
|
62
|
+ )
|
|
|
63
|
+
|
|
|
64
|
+ generate_info_table(html_file, baseline_info, benchmark_info)
|
|
|
65
|
+
|
|
|
66
|
+ # Generate total results table
|
|
|
67
|
+ generate_total_results_table(html_file, BASELINE_DIR, BENCHMARK_DIR)
|
|
|
68
|
+
|
|
|
69
|
+ # Generate results tables
|
|
|
70
|
+ for filename in os.listdir(BASELINE_DIR):
|
|
|
71
|
+ if filename.endswith(".txt") and not filename == "info.txt":
|
|
|
72
|
+ baseline_results = read_file(os.path.join(BASELINE_DIR, filename))
|
|
|
73
|
+ benchmark_results = read_file(os.path.join(BENCHMARK_DIR, filename))
|
|
|
74
|
+
|
|
|
75
|
+ generate_results_table(
|
|
|
76
|
+ html_file, baseline_results, benchmark_results, filename
|
|
|
77
|
+ )
|
|
|
78
|
+
|
|
|
79
|
+
|
|
|
80
|
+ write_to_html(html_file, "<center>Freetype Benchmark</center>\n")
|
|
|
81
|
+ write_to_html(html_file, "</body>\n</html>\n")
|
|
|
82
|
+
|
|
|
83
|
+def write_to_html(html_file, content):
|
|
|
84
|
+ """Write content to html file"""
|
|
|
85
|
+ html_file.write(content)
|
|
|
86
|
+
|
|
|
87
|
+
|
|
|
88
|
+def read_file(file_path):
|
|
|
89
|
+ """Read file and return list of lines"""
|
|
|
90
|
+ with open(file_path, "r") as f:
|
|
|
91
|
+ return f.readlines()
|
|
|
92
|
+
|
|
|
93
|
+
|
|
|
94
|
+def parse_info_file(info_file):
|
|
|
95
|
+ """Get info from info.txt file and return as list"""
|
|
|
96
|
+ info = read_file(info_file)
|
|
|
97
|
+ info[1] = '<a href="">"{}{}">{}</a>\n'.format(GITLAB_URL, info[1].strip(), info[1][:8])
|
|
|
98
|
+ return info
|
|
|
99
|
+
|
|
|
100
|
+
|
|
|
101
|
+def generate_info_table(html_file, baseline_info, benchmark_info):
|
|
|
102
|
+ """Prepare info table for html"""
|
|
|
103
|
+ write_to_html(html_file, "<h2>Info</h2>\n")
|
|
|
104
|
+ write_to_html(html_file, '<table border="1">\n')
|
|
|
105
|
+ write_to_html(
|
|
|
106
|
+ html_file, "<tr><th>Info</th><th>Baseline</th><th>Benchmark</th></tr>\n"
|
|
|
107
|
+ )
|
|
|
108
|
+ info_list = ["Parameters", "Commit ID", "Commit Date", "Branch"]
|
|
|
109
|
+ for info, baseline_line, benchmark_line in zip(
|
|
|
110
|
+ info_list, baseline_info, benchmark_info
|
|
|
111
|
+ ):
|
|
|
112
|
+ write_to_html(
|
|
|
113
|
+ html_file,
|
|
|
114
|
+ '<tr><td class="col1">{}</td><td>{}</td><td>{}</td></tr>\n'.format(
|
|
|
115
|
+ info, baseline_line.strip(), benchmark_line.strip()
|
|
|
116
|
+ ),
|
|
|
117
|
+ )
|
|
|
118
|
+ write_to_html(html_file, "</table><br/>")
|
|
|
119
|
+ write_to_html(html_file, f"<p>{INFO_1}</p>")
|
|
|
120
|
+ write_to_html(html_file, f"<p>{INFO_2}</p>")
|
|
|
121
|
+
|
|
|
122
|
+
|
|
|
123
|
+def generate_total_results_table(html_file, baseline_dir, benchmark_dir):
|
|
|
124
|
+ """Prepare total results table for html"""
|
|
|
125
|
+
|
|
|
126
|
+ # This dictionary will store aggregated results.
|
|
|
127
|
+ test_results = {test: {"baseline": 0, "benchmark": 0, "n_baseline": 0, "n_benchmark": 0} for test in [
|
|
|
128
|
+ "Load", "Load_Advances (Normal)", "Load_Advances (Fast)", "Load_Advances (Unscaled)", "Render",
|
|
|
129
|
+ "Get_Glyph", "Get_Char_Index", "Iterate CMap", "New_Face", "Embolden", "Stroke", "Get_BBox",
|
|
|
130
|
+ "Get_CBox", "New_Face & load glyph(s)"
|
|
|
131
|
+ ]}
|
|
|
132
|
+
|
|
|
133
|
+ total_time = 0
|
|
|
134
|
+
|
|
|
135
|
+ for filename in os.listdir(baseline_dir):
|
|
|
136
|
+ if filename.endswith(".txt") and not filename == "info.txt":
|
|
|
137
|
+
|
|
|
138
|
+ baseline_results = read_file(os.path.join(baseline_dir, filename))
|
|
|
139
|
+ benchmark_results = read_file(os.path.join(benchmark_dir, filename))
|
|
|
140
|
+
|
|
|
141
|
+ for baseline_line, benchmark_line in zip(baseline_results, benchmark_results):
|
|
|
142
|
+
|
|
|
143
|
+ if baseline_line.startswith("Total time:"):
|
|
|
144
|
+ baseline_match = re.match(r"Total time: (\d+)s", baseline_line)
|
|
|
145
|
+ benchmark_match = re.match(r"Total time: (\d+)s", benchmark_line)
|
|
|
146
|
+
|
|
|
147
|
+ if baseline_match and benchmark_match:
|
|
|
148
|
+ total_time += int(baseline_match.group(1))
|
|
|
149
|
+ total_time += int(benchmark_match.group(1))
|
|
|
150
|
+
|
|
|
151
|
+
|
|
|
152
|
+ if baseline_line.startswith(" "):
|
|
|
153
|
+ baseline_match = re.match(r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", baseline_line)
|
|
|
154
|
+ benchmark_match = re.match(r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", benchmark_line)
|
|
|
155
|
+
|
|
|
156
|
+ if baseline_match and benchmark_match:
|
|
|
157
|
+ test = baseline_match.group(1).strip()
|
|
|
158
|
+ baseline_value = float(baseline_match.group(2))
|
|
|
159
|
+ benchmark_value = float(benchmark_match.group(2))
|
|
|
160
|
+ baseline_n = int(baseline_match.group(3))
|
|
|
161
|
+ benchmark_n = int(benchmark_match.group(3))
|
|
|
162
|
+
|
|
|
163
|
+ # Aggregate the results
|
|
|
164
|
+ if test in test_results:
|
|
|
165
|
+ test_results[test]["baseline"] += baseline_value
|
|
|
166
|
+ test_results[test]["benchmark"] += benchmark_value
|
|
|
167
|
+ test_results[test]["n_baseline"] += baseline_n
|
|
|
168
|
+ test_results[test]["n_benchmark"] += benchmark_n
|
|
|
169
|
+
|
|
|
170
|
+
|
|
|
171
|
+
|
|
|
172
|
+ # Writing to HTML
|
|
|
173
|
+ write_to_html(html_file, "<h2>Total Results</h2>\n")
|
|
|
174
|
+ write_to_html(html_file, '<table border="1">\n')
|
|
|
175
|
+ write_to_html(
|
|
|
176
|
+ html_file,
|
|
|
177
|
+ '<tr><th>Test</th><th>Iterations</th><th>* Baseline (µs)</th>\
|
|
|
178
|
+ <th>* Benchmark (µs)</th><th>Difference (%)</th></tr>\n'
|
|
|
179
|
+ )
|
|
|
180
|
+
|
|
|
181
|
+ total_baseline = total_benchmark = total_diff = total_n_baseline = total_n_benchmark = 0
|
|
|
182
|
+
|
|
|
183
|
+ for test, values in test_results.items():
|
|
|
184
|
+
|
|
|
185
|
+ baseline = values["baseline"] / FONT_COUNT
|
|
|
186
|
+ benchmark = values["benchmark"] / FONT_COUNT
|
|
|
187
|
+ n_baseline = values["n_baseline"] / FONT_COUNT
|
|
|
188
|
+ n_benchmark = values["n_benchmark"] / FONT_COUNT
|
|
|
189
|
+
|
|
|
190
|
+ n_display = f"{n_baseline:.0f} | {n_benchmark:.0f}" if n_baseline != n_benchmark else int(n_baseline)
|
|
|
191
|
+
|
|
|
192
|
+ diff = ((baseline - benchmark) / baseline) * 100
|
|
|
193
|
+
|
|
|
194
|
+ # Calculate for total row
|
|
|
195
|
+ total_baseline += baseline
|
|
|
196
|
+ total_benchmark += benchmark
|
|
|
197
|
+ total_n_baseline += n_baseline
|
|
|
198
|
+ total_n_benchmark += n_benchmark
|
|
|
199
|
+
|
|
|
200
|
+ # Check which value is smaller for color highlighting
|
|
|
201
|
+ baseline_color = "highlight" if baseline <= benchmark else ""
|
|
|
202
|
+ benchmark_color = "highlight" if benchmark <= baseline else ""
|
|
|
203
|
+
|
|
|
204
|
+ write_to_html(
|
|
|
205
|
+ html_file,
|
|
|
206
|
+ f'<tr><td class="col1">{test}</td><td>{n_display}</td>\
|
|
|
207
|
+ <td class="{baseline_color}">{baseline:.1f}</td>\
|
|
|
208
|
+ <td class="{benchmark_color}">{benchmark:.1f}</td><td>{diff:.1f}</td></tr>\n'
|
|
|
209
|
+ )
|
|
|
210
|
+
|
|
|
211
|
+
|
|
|
212
|
+
|
|
|
213
|
+ total_diff = ((total_baseline - total_benchmark) / total_baseline) * 100
|
|
|
214
|
+ total_n_display = f"{total_n_baseline} | {total_n_benchmark}" if total_n_baseline != total_n_benchmark else str(total_n_baseline)
|
|
|
215
|
+
|
|
|
216
|
+ write_to_html(
|
|
|
217
|
+ html_file,
|
|
|
218
|
+ f'<tr><td class="col1">Total duration for all tests:</td><td class="col1" colspan="4">{total_time:.0f} s</td>'
|
|
|
219
|
+ )
|
|
|
220
|
+
|
|
|
221
|
+ write_to_html(html_file,'</table>\n')
|
|
|
222
|
+
|
|
|
223
|
+
|
|
|
224
|
+
|
|
|
225
|
+def generate_results_table(html_file, baseline_results, benchmark_results, filename):
|
|
|
226
|
+ """Prepare results table for html"""
|
|
|
227
|
+ fontname = [
|
|
|
228
|
+ line.split("/")[-1].strip("'")[:-2]
|
|
|
229
|
+ for line in baseline_results
|
|
|
230
|
+ if line.startswith("ftbench results for font")
|
|
|
231
|
+ ][0]
|
|
|
232
|
+
|
|
|
233
|
+ write_to_html(html_file, "<h3>Results for {}</h2>\n".format(fontname))
|
|
|
234
|
+ write_to_html(html_file, '<table border="1">\n')
|
|
|
235
|
+ write_to_html(
|
|
|
236
|
+ html_file,
|
|
|
237
|
+ '<tr><th>Test</th><th>Iterations</th>\
|
|
|
238
|
+ <th>* <a href="">"{}.txt">Baseline</a> (µs)</th>\
|
|
|
239
|
+ <th>* <a href="">"{}.txt">Benchmark</a> (µs)</th>\
|
|
|
240
|
+ <th>Difference (%)</th></tr>\n'.format(
|
|
|
241
|
+ os.path.join("./baseline/", filename[:-4]),
|
|
|
242
|
+ os.path.join("./benchmark/", filename[:-4]),
|
|
|
243
|
+ ),
|
|
|
244
|
+ )
|
|
|
245
|
+
|
|
|
246
|
+ total_n = total_difference = total_time = 0
|
|
|
247
|
+
|
|
|
248
|
+ for baseline_line, benchmark_line in zip(baseline_results, benchmark_results):
|
|
|
249
|
+
|
|
|
250
|
+ if baseline_line.startswith("Total time:"):
|
|
|
251
|
+ baseline_match = re.match(r"Total time: (\d+)s", baseline_line)
|
|
|
252
|
+ benchmark_match = re.match(r"Total time: (\d+)s", benchmark_line)
|
|
|
253
|
+
|
|
|
254
|
+ if baseline_match and benchmark_match:
|
|
|
255
|
+ total_time += int(baseline_match.group(1))
|
|
|
256
|
+ total_time += int(benchmark_match.group(1))
|
|
|
257
|
+
|
|
|
258
|
+
|
|
|
259
|
+ if baseline_line.startswith(" "):
|
|
|
260
|
+ baseline_match = re.match(
|
|
|
261
|
+ r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", baseline_line
|
|
|
262
|
+ )
|
|
|
263
|
+ benchmark_match = re.match(
|
|
|
264
|
+ r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", benchmark_line
|
|
|
265
|
+ )
|
|
|
266
|
+
|
|
|
267
|
+ if baseline_match and benchmark_match:
|
|
|
268
|
+ baseline_value = float(baseline_match.group(2))
|
|
|
269
|
+ benchmark_value = float(benchmark_match.group(2))
|
|
|
270
|
+
|
|
|
271
|
+ percentage_diff = (
|
|
|
272
|
+ (baseline_value - benchmark_value) / baseline_value
|
|
|
273
|
+ ) * 100
|
|
|
274
|
+
|
|
|
275
|
+ baseline_n = baseline_match.group(3)
|
|
|
276
|
+ benchmark_n = benchmark_match.group(3)
|
|
|
277
|
+
|
|
|
278
|
+
|
|
|
279
|
+ n = (
|
|
|
280
|
+ baseline_n
|
|
|
281
|
+ if baseline_n == benchmark_n
|
|
|
282
|
+ else baseline_n + " | " + benchmark_n
|
|
|
283
|
+ )
|
|
|
284
|
+
|
|
|
285
|
+
|
|
|
286
|
+
|
|
|
287
|
+ total_n += int(baseline_n)
|
|
|
288
|
+ total_n += int(benchmark_n)
|
|
|
289
|
+
|
|
|
290
|
+
|
|
|
291
|
+ # Check which value is smaller for color highlighting
|
|
|
292
|
+ baseline_color = "highlight" if baseline_value <= benchmark_value else ""
|
|
|
293
|
+ benchmark_color = "highlight" if benchmark_value <= baseline_value else ""
|
|
|
294
|
+
|
|
|
295
|
+
|
|
|
296
|
+ write_to_html(
|
|
|
297
|
+ html_file,
|
|
|
298
|
+ f'<tr><td class="col1">{baseline_match.group(1)}</td><td>{n}</td>\
|
|
|
299
|
+ <td class="{baseline_color}">{baseline_value:.1f}</td><td class="{benchmark_color}">{benchmark_value:.1f}</td><td>{percentage_diff:.1f}</td></tr>\n'
|
|
|
300
|
+ )
|
|
|
301
|
+
|
|
|
302
|
+ write_to_html(
|
|
|
303
|
+ html_file,
|
|
|
304
|
+ f'<tr><td class="col1">Total duration for the font:</td><td class="col1" colspan="4">{total_time:.0f} s</td></table>\n'
|
|
|
305
|
+ )
|
|
|
306
|
+
|
|
|
307
|
+
|
|
|
308
|
+
|
|
|
309
|
+if __name__ == "__main__":
|
|
|
310
|
+ main() |