mirror of
https://github.com/isc-projects/bind9.git
synced 2026-05-22 18:17:05 -04:00
Move utility Python scripts to the bind9-qa repo
I would have touched them anyway in this MR so let's bite the bullet and move them there.
This commit is contained in:
Štěpán Balážik
parent
482c1cc72f
commit
a3d0f43d21
5 changed files with 11 additions and 301 deletions
|
|
@ -396,7 +396,7 @@ stages:
|
|||
|
||||
.parse_tsan: &parse_tsan
|
||||
- *find_python
|
||||
- find -name 'tsan.*' -exec "$PYTHON" util/parse_tsan.py {} \;
|
||||
- find -name 'tsan.*' -exec "$PYTHON" "$CI_PROJECT_DIR"/bind9-qa/ci/parse_tsan.py {} \;
|
||||
|
||||
.check_readline_setup: &check_readline_setup
|
||||
- if [[ -n "${WITHOUT_LIBEDIT}" ]]; then
|
||||
|
|
@ -510,7 +510,9 @@ stages:
|
|||
-F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN"
|
||||
-F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MAX]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MAX"
|
||||
https://gitlab.isc.org/api/v4/projects/188/trigger/pipeline | jq .id)
|
||||
- util/ci-wait-shotgun.py $PIPELINE_ID
|
||||
- *git_clone_bind9-qa
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/ci/wait_shotgun.py $PIPELINE_ID
|
||||
needs:
|
||||
- job: ci-variables
|
||||
artifacts: true
|
||||
|
|
@ -531,12 +533,12 @@ stages:
|
|||
- *fips_feature_test
|
||||
- *find_pytest
|
||||
- *find_python
|
||||
- ( if [ "${CI_DISPOSABLE_ENVIRONMENT}" = "true" ]; then sleep 3000; "$PYTHON" "${CI_PROJECT_DIR}/util/get-running-system-tests.py"; fi ) &
|
||||
- *git_clone_bind9-qa
|
||||
- ( if [ "${CI_DISPOSABLE_ENVIRONMENT}" = "true" ]; then sleep 3000; "$PYTHON" "${CI_PROJECT_DIR}/bind9-qa/ci/get_running_system_tests.py"; fi ) &
|
||||
- cd bin/tests/system
|
||||
- RET=0
|
||||
- >
|
||||
("$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit_pytest.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt) || RET=1
|
||||
- *git_clone_bind9-qa
|
||||
- >
|
||||
"$PYTHON" "$CI_PROJECT_DIR"/bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit_pytest.xml --output "$CI_PROJECT_DIR"/junit.xml
|
||||
- (exit $RET)
|
||||
|
|
@ -688,7 +690,7 @@ ci-orphaned-anchors:
|
|||
script:
|
||||
- *git_clone_bind9-qa
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/ci-orphaned-anchors/check-orphaned-anchors-ci.py .gitlab-ci.yml
|
||||
"$CI_PROJECT_DIR"/bind9-qa/ci-orphaned-anchors/check_orphaned_anchors_ci.py .gitlab-ci.yml
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
||||
changes:
|
||||
|
|
@ -825,7 +827,7 @@ generate-stress-test-configs:
|
|||
script:
|
||||
- *git_clone_bind9-qa
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/stress/generate-stress-test-configs.py > stress-test-configs.yml
|
||||
"$CI_PROJECT_DIR"/bind9-qa/stress/generate_stress_test_configs.py > stress-test-configs.yml
|
||||
artifacts:
|
||||
paths:
|
||||
- stress-test-configs.yml
|
||||
|
|
@ -833,7 +835,9 @@ generate-stress-test-configs:
|
|||
generate-tsan-stress-test-configs:
|
||||
<<: *quick_checks_job
|
||||
script:
|
||||
- util/generate-tsan-stress-jobs.py > tsan-stress-test-configs.yml
|
||||
- *git_clone_bind9-qa
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/ci/generate_tsan_stress_jobs.py > tsan-stress-test-configs.yml
|
||||
artifacts:
|
||||
paths:
|
||||
- tsan-stress-test-configs.yml
|
||||
|
|
|
|||
|
|
@ -1,106 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
|
||||
#
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# See the COPYRIGHT file distributed with this work for additional
|
||||
# information regarding copyright ownership.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import gitlab
|
||||
|
||||
|
||||
def init_gitlab_project():
|
||||
if os.getenv("CI_SERVER_URL", None) is not None:
|
||||
url = os.getenv("CI_SERVER_URL")
|
||||
job_token = os.environ["CI_JOB_TOKEN"]
|
||||
gl = gitlab.Gitlab(url, job_token=job_token)
|
||||
else: # running locally on dev machine
|
||||
gl = gitlab.Gitlab.from_config()
|
||||
return gl.projects.get("isc-projects/bind9-shotgun-ci")
|
||||
|
||||
|
||||
def parse_parent_pipeline_id():
|
||||
if len(sys.argv) != 2:
|
||||
raise RuntimeError("usage: util/ci-wait-shotgun.py PIPELINE_ID")
|
||||
try:
|
||||
return int(sys.argv[1])
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("error: PIPELINE_ID must be a number") from exc
|
||||
|
||||
|
||||
def wait_until(callback, timeout=1800, retry=10):
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
if callback():
|
||||
return
|
||||
time.sleep(retry)
|
||||
raise RuntimeError(f"error: timed out after {timeout}s")
|
||||
|
||||
|
||||
def get_child_pipeline_id(project, pipeline_id):
|
||||
pipeline = project.pipelines.get(pipeline_id)
|
||||
|
||||
def pipeline_finished():
|
||||
pipeline.refresh()
|
||||
return pipeline.finished_at is not None
|
||||
|
||||
wait_until(pipeline_finished, timeout=6600)
|
||||
|
||||
bridges = pipeline.bridges.list()
|
||||
if len(bridges) != 1:
|
||||
raise RuntimeError("error: expected exactly one child pipeline")
|
||||
return bridges[0].downstream_pipeline["id"]
|
||||
|
||||
|
||||
def get_postproc_job(project, pipeline_id):
|
||||
pipeline = project.pipelines.get(pipeline_id)
|
||||
postproc_job = None
|
||||
|
||||
def job_finished():
|
||||
nonlocal postproc_job
|
||||
for job in pipeline.jobs.list(get_all=True):
|
||||
if job.name == "postproc":
|
||||
postproc_job = job
|
||||
if postproc_job is None:
|
||||
raise RuntimeError("error: failed to find 'postproc' job in child pipeline")
|
||||
return postproc_job.finished_at is not None
|
||||
|
||||
wait_until(job_finished)
|
||||
return postproc_job
|
||||
|
||||
|
||||
def evaluate_postproc_job(job):
|
||||
print(f"postproc job URL: {job.web_url}")
|
||||
index_url = (
|
||||
"https://isc-projects.gitlab-pages.isc.org/-/"
|
||||
f"bind9-shotgun-ci/-/jobs/{job.id}/artifacts/index.html"
|
||||
)
|
||||
if job.status in ["success", "failed"]:
|
||||
print(f"result for manual inspection: {index_url}")
|
||||
if job.status != "success":
|
||||
raise RuntimeError("error: 'postproc' job didn't succeed")
|
||||
|
||||
|
||||
def main():
|
||||
project = init_gitlab_project()
|
||||
parent_pipeline_id = parse_parent_pipeline_id()
|
||||
child_pipeline_id = get_child_pipeline_id(project, parent_pipeline_id)
|
||||
postproc_job = get_postproc_job(project, child_pipeline_id)
|
||||
evaluate_postproc_job(postproc_job)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except RuntimeError as err:
|
||||
print(err)
|
||||
sys.exit(1)
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
|
||||
#
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# See the COPYRIGHT file distributed with this work for additional
|
||||
# information regarding copyright ownership.
|
||||
|
||||
import yaml
|
||||
|
||||
NUMBER_OF_TESTS_PER_TSAN_JOB = 50
|
||||
|
||||
with open(".gitlab-ci.yml", encoding="utf-8") as gitlab_ci_yml:
|
||||
anchors = yaml.load(gitlab_ci_yml, Loader=yaml.Loader)
|
||||
|
||||
for tsan_job in "gcc:tsan", "clang:tsan":
|
||||
for test_type in "unit", "system":
|
||||
tsan_stress_test_job = anchors[f"{test_type}:{tsan_job}"]
|
||||
tsan_stress_test_job["stage"] = "test"
|
||||
tsan_stress_test_job["rules"] = [
|
||||
{"if": '$CI_PIPELINE_SOURCE == "parent_pipeline"'}
|
||||
]
|
||||
tsan_stress_test_job["parallel"] = NUMBER_OF_TESTS_PER_TSAN_JOB
|
||||
tsan_stress_test_job["needs"] = [
|
||||
{"pipeline": "$PARENT_PIPELINE_ID", "job": tsan_job}
|
||||
]
|
||||
|
||||
print(
|
||||
yaml.dump(
|
||||
{f"{test_type}:{tsan_job}:stress": tsan_stress_test_job},
|
||||
Dumper=yaml.Dumper,
|
||||
)
|
||||
)
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
|
||||
#
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# See the COPYRIGHT file distributed with this work for additional
|
||||
# information regarding copyright ownership.
|
||||
|
||||
import psutil
|
||||
|
||||
for pid in psutil.pids():
|
||||
try:
|
||||
environ = psutil.Process(pid).environ()
|
||||
if "PYTEST_CURRENT_TEST" in environ:
|
||||
name = psutil.Process(pid).name()
|
||||
print(
|
||||
f'pytest process {name}/{pid} running: {environ["PYTEST_CURRENT_TEST"]}'
|
||||
)
|
||||
except (psutil.AccessDenied, psutil.NoSuchProcess):
|
||||
pass
|
||||
|
|
@ -1,125 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
############################################################################
|
||||
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
|
||||
#
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# See the COPYRIGHT file distributed with this work for additional
|
||||
# information regarding copyright ownership.
|
||||
############################################################################
|
||||
|
||||
"""Parse the ThreadSanizer reports, unify them and put them into unique dirs."""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
from hashlib import sha256
|
||||
|
||||
|
||||
class State:
|
||||
"""Class that holds state of the TSAN parser."""
|
||||
|
||||
inside = False
|
||||
block = ""
|
||||
last_line = None
|
||||
|
||||
mutexes = {}
|
||||
m_index = 1
|
||||
threads = {}
|
||||
t_index = 1
|
||||
pointers = {}
|
||||
p_index = 1
|
||||
|
||||
def __init__(self):
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
"""Reset the object to initial state"""
|
||||
|
||||
self.inside = False
|
||||
self.block = ""
|
||||
|
||||
self.mutexes = {}
|
||||
self.threads = {}
|
||||
self.pointers = {}
|
||||
self.pointers["0x000000000000"] = 0
|
||||
|
||||
self.m_index = 1
|
||||
self.t_index = 1
|
||||
self.p_index = 1
|
||||
|
||||
|
||||
TOP = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
|
||||
OUT = os.path.join(TOP, "tsan")
|
||||
|
||||
if not os.path.isdir(OUT):
|
||||
os.mkdir(OUT)
|
||||
|
||||
# Regular Expressions
|
||||
MUTEX = re.compile(r"M\d+")
|
||||
THREAD = re.compile(r"T\d+")
|
||||
STACK = re.compile(r"\s\(\S+\+0x\S+\)")
|
||||
POINTER = re.compile(r"0x[0-9a-f]+")
|
||||
PID = re.compile(r"\(pid=\d+,?\)")
|
||||
TID = re.compile(r"tid=\d+,?\s*")
|
||||
WORKER = re.compile(r"\s+'(isc-worker|isc-net-)\d+'")
|
||||
PATH = re.compile(TOP + "/")
|
||||
|
||||
S = State()
|
||||
|
||||
with open(sys.argv[1], "r", encoding="utf-8") as f:
|
||||
for line in f.readlines():
|
||||
if line == "==================\n":
|
||||
if not S.inside:
|
||||
S.inside = True
|
||||
else:
|
||||
DNAME = sha256(S.last_line.encode("utf-8")).hexdigest()
|
||||
DNAME = os.path.join(OUT, DNAME)
|
||||
if not os.path.isdir(DNAME):
|
||||
os.mkdir(DNAME)
|
||||
FNAME = sha256(S.block.encode("utf-8")).hexdigest() + ".txt"
|
||||
FNAME = os.path.join(DNAME, FNAME)
|
||||
if not os.path.isfile(FNAME):
|
||||
with open(FNAME, "w", encoding="utf-8") as w:
|
||||
w.write(S.block)
|
||||
S.reset()
|
||||
else:
|
||||
for m in MUTEX.finditer(line):
|
||||
k = m.group()
|
||||
if k not in S.mutexes:
|
||||
S.mutexes[k] = S.m_index
|
||||
S.m_index += 1
|
||||
for m in THREAD.finditer(line):
|
||||
k = m.group()
|
||||
if k not in S.threads:
|
||||
S.threads[k] = S.t_index
|
||||
S.t_index += 1
|
||||
for m in POINTER.finditer(line):
|
||||
k = m.group()
|
||||
if k not in S.pointers:
|
||||
S.pointers[k] = S.p_index
|
||||
S.p_index += 1
|
||||
for k, v in S.mutexes.items():
|
||||
r = re.compile(k)
|
||||
line = r.sub(f"M{v:04d}", line)
|
||||
for k, v in S.threads.items():
|
||||
r = re.compile(k)
|
||||
line = r.sub(f"T{v:04d}", line)
|
||||
for k, v in S.pointers.items():
|
||||
r = re.compile(k)
|
||||
line = r.sub(f"0x{v:012d}", line)
|
||||
|
||||
line = STACK.sub("", line)
|
||||
line = PID.sub("", line)
|
||||
line = TID.sub("", line)
|
||||
line = WORKER.sub("", line)
|
||||
line = PATH.sub("", line)
|
||||
|
||||
S.block += line
|
||||
S.last_line = line
|
||||
Loading…
Reference in a new issue