Skip to content
Permalink
Newer
Older
100755 317 lines (264 sloc) 11 KB
1
#!/usr/bin/env python3
2
import os
3
import subprocess
4
import sys
5
import xml.etree.ElementTree as ET
6
import json
7
import argparse
8
import shlex
9
10
REPORT_HEADER = """<?xml version="1.0" encoding="UTF-8"?>
11
<?xml-stylesheet xmlns="http://www.w3.org/1999/xhtml" type="text/xsl" href="#stylesheet"?>
12
<!DOCTYPE Site [
13
<!ATTLIST ns0:stylesheet
14
id ID #REQUIRED>
15
]>
16
"""
17
18
19
def handle_args(argv):
20
"""
21
Handles the arguements to the script.
22
"""
23
parser = argparse.ArgumentParser()
24
parser.add_argument(
25
'--cmake-exe',
26
help='Name of the CMake executable',
27
type=str,
28
default='cmake')
29
parser.add_argument(
30
'-a',
31
'--additional-cmake-args',
32
help='Additional args to hand to CMake required by the tested implementation.',
33
type=str)
34
parser.add_argument(
35
'-b',
36
'--build-system-name',
37
help='The name of the build system as known by CMake, for example \'Ninja\'.',
38
type=str,
39
required=True)
40
parser.add_argument(
41
'-c',
42
'--build-system-call',
43
help='The call to the used build system.',
44
type=str,
45
required=True)
46
parser.add_argument(
47
'--build-only',
48
help='Whether to perform only a build without any testing.',
49
required=False,
50
action='store_true')
51
parser.add_argument(
52
'-e',
53
'--exclude-categories',
54
help='List of test categories to',
57
parser.add_argument(
58
'--fast',
59
help='Disable full conformance mode to avoid extensive tests.',
60
required=False,
61
action='store_true')
62
parser.add_argument(
63
'--disable-deprecated-features',
64
help='Disable tests for the deprecated SYCL features.',
65
required=False,
66
action='store_true')
68
'--device',
69
help='Select SYCL device to run CTS on. ECMAScript regular expression syntax can be used.',
70
type=str,
71
required=True)
72
parser.add_argument(
73
'-n',
74
'--implementation-name',
75
help='The name of the implementation to be displayed in the report.',
76
type=str,
77
required=True)
78
parser.add_argument('--additional-ctest-args',
79
'--ctest-args',
80
help='Additional args to hand to CTest.',
81
type=str)
82
args = parser.parse_args(argv)
83
84
full_conformance = 'OFF' if args.fast else 'ON'
85
test_deprecated_features = 'OFF' if args.disable_deprecated_features else 'ON'
86
87
return (args.cmake_exe, args.build_system_name, args.build_system_call,
88
full_conformance, test_deprecated_features, args.exclude_categories,
89
args.implementation_name, args.additional_cmake_args, args.device,
90
args.additional_ctest_args, args.build_only)
93
def split_additional_args(additional_args):
94
"""
95
Split any 'additional argument' parameter passed to the script into the list
96
"""
97
98
# shlex doesn't support None
99
if additional_args is None:
100
return []
101
use_posix_mode = os.name != 'nt'
102
# shlex was not intended for Windows by design, with non-POSIX Unix shells
103
# supported. Still it provides a Windows-compilant solution up to the
104
# certain degree, details: https://bugs.python.org/issue1724822
105
# The rules for quoting and escaping in non-POSIX mode are different from
106
# the native Windows rules, defined for CommandLineToArgvW for example.
107
# Still it makes possible to use something like "%ENV_VAR%\subdir" within
108
# additional arguments
109
return shlex.split(additional_args, posix=use_posix_mode)
112
def generate_cmake_call(cmake_exe, build_system_name, full_conformance,
113
test_deprecated_features, exclude_categories,
114
additional_cmake_args, device):
115
"""
116
Generates a CMake call based on the input in a form accepted by
117
subprocess.call().
118
"""
120
cmake_exe,
121
'..',
122
'-G' + build_system_name,
123
'-DSYCL_CTS_ENABLE_FULL_CONFORMANCE=' + full_conformance,
124
'-DSYCL_CTS_ENABLE_DEPRECATED_FEATURES_TESTS=' + test_deprecated_features,
125
'-DSYCL_CTS_CTEST_DEVICE=' + device,
126
]
127
if exclude_categories is not None:
128
call += ['-DSYCL_CTS_EXCLUDE_TEST_CATEGORIES=' + exclude_categories]
129
call += split_additional_args(additional_cmake_args)
130
return call
133
def generate_ctest_call(additional_ctest_args):
134
"""
135
Generates a CTest call based on the input in a form accepted by
136
subprocess.call().
137
"""
138
return [
139
'ctest', '.', '-T', 'Test', '--no-compress-output',
140
'--test-output-size-passed', '0', '--test-output-size-failed', '0'
141
] + split_additional_args(additional_ctest_args)
144
def subprocess_call(parameter_list):
145
"""
146
Calls subprocess.call() with the parameter list.
147
Prints the invocation before doing the call.
148
"""
149
print("subprocess.call:\n %s" % " ".join(parameter_list))
150
return subprocess.call(parameter_list)
151
152
153
def configure_and_run_tests(cmake_call, build_system_call, build_only,
154
ctest_call):
155
"""
156
Configures the tests with cmake to produce a ninja.build file.
157
Runs the generated ninja file.
158
Runs ctest, overwriting any cached results.
159
"""
160
161
build_system_call = build_system_call.split()
162
163
subprocess_call(cmake_call)
164
error_code = subprocess_call(build_system_call)
165
if (not build_only):
166
error_code = subprocess_call(ctest_call)
167
return error_code
168
169
170
def collect_info_filenames():
171
"""
172
Collects all the .info test result files in the Testing directory.
173
Exits the program if no result files are found.
174
"""
175
176
info_filenames = []
177
178
# Get all the test results in Testing
179
for filename in os.listdir('Testing'):
180
filename_full = os.path.join('Testing', filename)
181
if filename.endswith('.info'):
182
info_filenames.append(filename_full)
183
184
# Exit if we didn't find any test results
185
if (len(info_filenames) == 0):
186
print("Fatal error: Could not find any device info dumps")
189
return info_filenames
192
def get_valid_json_info(info_filenames):
194
Ensures that all the .info files have the same data, then returns the
195
parsed json.
196
"""
197
198
reference_info = None
199
for info_file in info_filenames:
200
with open(info_file, 'r') as info:
201
if reference_info is None:
202
reference_info = info.read()
203
elif info.read() != reference_info:
204
print('Fatal error: mismatch in device info dumps between tests')
207
return json.loads(reference_info)
208
209
210
def get_xml_test_results():
211
"""
212
Finds the xml file output by the test and returns the rool of the xml tree.
213
"""
214
test_tag = ""
215
with open(os.path.join("Testing", "TAG"), 'r') as tag_file:
216
test_tag = tag_file.readline()[:-1]
217
218
test_xml_file = os.path.join("Testing", test_tag, "Test.xml")
219
test_xml_tree = ET.parse(test_xml_file)
220
return test_xml_tree.getroot()
221
222
223
def update_xml_attribs(info_json, implementation_name, test_xml_root,
224
full_conformance, cmake_call, build_system_name,
225
build_system_call, ctest_call):
226
"""
227
Adds attributes to the root of the xml trees json required by the
228
conformance report.
229
These attributes describe the device and platform information used in the
230
tests, along with the configuration and execution details of the tests.
231
"""
232
233
# Set Host Device Information attribs
234
test_xml_root.attrib["BuildName"] = implementation_name
235
test_xml_root.attrib["PlatformName"] = info_json['platform-name']
236
test_xml_root.attrib["PlatformVendor"] = info_json[
238
test_xml_root.attrib["PlatformVersion"] = info_json[
240
test_xml_root.attrib["DeviceName"] = info_json['device-name']
241
test_xml_root.attrib["DeviceVendor"] = info_json['device-vendor']
242
test_xml_root.attrib["DeviceVersion"] = info_json[
244
test_xml_root.attrib["DeviceType"] = info_json['device-type']
246
# Set Device Extension Support attribs
247
# TODO: Revisit this for SYCL 2020 aspects
248
test_xml_root.attrib["DeviceFP16"] = info_json['device-fp16']
249
test_xml_root.attrib["DeviceFP64"] = info_json['device-fp64']
250
251
# Set Build Information attribs
252
test_xml_root.attrib["FullConformanceMode"] = full_conformance
253
test_xml_root.attrib["CMakeInput"] = ' '.join(cmake_call)
254
test_xml_root.attrib["BuildSystemGenerator"] = build_system_name
255
test_xml_root.attrib["BuildSystemCall"] = build_system_call
256
test_xml_root.attrib["CTestCall"] = ' '.join(ctest_call)
257
258
return test_xml_root
259
260
261
def main(argv=sys.argv[1:]):
262
263
# Parse and gather all the script args
264
(cmake_exe, build_system_name, build_system_call, full_conformance,
265
test_deprecated_features, exclude_categories, implementation_name,
266
additional_cmake_args, device, additional_ctest_args,
267
build_only) = handle_args(argv)
268
269
# Generate a cmake call in a form accepted by subprocess.call()
270
cmake_call = generate_cmake_call(cmake_exe, build_system_name,
271
full_conformance, test_deprecated_features,
272
exclude_categories, additional_cmake_args,
273
device)
275
# Generate a CTest call in a form accepted by subprocess.call()
276
ctest_call = generate_ctest_call(additional_ctest_args)
277
278
# Make a build directory if required and enter it
279
if not os.path.isdir('build'):
280
os.mkdir('build')
281
os.chdir('build')
282
283
# Configure the build system with cmake, run the build, and run the tests.
284
error_code = configure_and_run_tests(cmake_call, build_system_call,
285
build_only, ctest_call)
286
287
if build_only:
288
return error_code
289
290
# Collect the test info files, validate them and get the contents as json.
291
info_filenames = collect_info_filenames()
292
info_json = get_valid_json_info(info_filenames)
293
294
# Get the xml results and update with the necessary information.
295
result_xml_root = get_xml_test_results()
296
result_xml_root = update_xml_attribs(info_json, implementation_name,
297
result_xml_root, full_conformance,
298
cmake_call, build_system_name,
299
build_system_call, ctest_call)
300
301
# Get the xml report stylesheet and add it to the results.
302
stylesheet_xml_file = os.path.join("..", "tools", "stylesheet.xml")
303
stylesheet_xml_tree = ET.parse(stylesheet_xml_file)
304
stylesheet_xml_root = stylesheet_xml_tree.getroot()
305
result_xml_root.append(stylesheet_xml_root[0])
306
307
# Get the xml results as a string and append them to the report header.
308
report = REPORT_HEADER + ET.tostring(result_xml_root).decode("utf-8")
309
310
with open("conformance_report.xml", 'w') as final_conformance_report:
311
final_conformance_report.write(report)
312
313
return error_code
314
315
316
if __name__ == "__main__":
317
main()