mirror of
https://github.com/isc-projects/bind9.git
synced 2026-05-28 04:34:54 -04:00
[9.18] chg: ci: Move and/or rename Python scripts
Move some scripts to the QA repo, rename others to adhere to the snake-case Python convention. Partial backport of MR !11499 Merge branch 'backport-stepan/python-tooling-9.18' into 'bind-9.18' See merge request isc-projects/bind9!11575
This commit is contained in:
Štěpán Balážik
commit
9e23fcf74e
10 changed files with 39 additions and 321 deletions
|
|
@ -366,7 +366,7 @@ stages:
|
|||
|| (test -s config.log && cat config.log; exit 1)
|
||||
|
||||
.git-clone-bind9-qa: &git_clone_bind9-qa
|
||||
- git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
|
||||
- git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git "$CI_PROJECT_DIR"/bind9-qa
|
||||
|
||||
# change directory to the workspace before including this
|
||||
.find_python: &find_python
|
||||
|
|
@ -379,7 +379,7 @@ stages:
|
|||
|
||||
.parse_tsan: &parse_tsan
|
||||
- *find_python
|
||||
- find -name 'tsan.*' -exec "$PYTHON" util/parse_tsan.py {} \;
|
||||
- find -name 'tsan.*' -exec "$PYTHON" "$CI_PROJECT_DIR"/bind9-qa/ci/parse_tsan.py {} \;
|
||||
|
||||
.check_readline_setup: &check_readline_setup
|
||||
- if [[ -n "${WITHOUT_READLINE}" ]]; then
|
||||
|
|
@ -493,7 +493,9 @@ stages:
|
|||
-F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN"
|
||||
-F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MAX]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MAX"
|
||||
https://gitlab.isc.org/api/v4/projects/188/trigger/pipeline | jq .id)
|
||||
- util/ci-wait-shotgun.py $PIPELINE_ID
|
||||
- *git_clone_bind9-qa
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/ci/wait_shotgun.py $PIPELINE_ID
|
||||
needs:
|
||||
- job: ci-variables
|
||||
artifacts: true
|
||||
|
|
@ -514,14 +516,14 @@ stages:
|
|||
script:
|
||||
- *find_pytest
|
||||
- *find_python
|
||||
- ( if [ "${CI_DISPOSABLE_ENVIRONMENT}" = "true" ]; then sleep 3000; "$PYTHON" "${CI_PROJECT_DIR}/util/get-running-system-tests.py"; fi ) &
|
||||
- *git_clone_bind9-qa
|
||||
- ( if [ "${CI_DISPOSABLE_ENVIRONMENT}" = "true" ]; then sleep 3000; "$PYTHON" "${CI_PROJECT_DIR}/bind9-qa/ci/get_running_system_tests.py"; fi ) &
|
||||
- cd bin/tests/system
|
||||
- RET=0
|
||||
- >
|
||||
("$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit_pytest.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt) || RET=1
|
||||
- *git_clone_bind9-qa
|
||||
- >
|
||||
"$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit_pytest.xml --output "$CI_PROJECT_DIR"/junit.xml
|
||||
"$PYTHON" "$CI_PROJECT_DIR"/bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit_pytest.xml --output "$CI_PROJECT_DIR"/junit.xml
|
||||
- (exit $RET)
|
||||
- '( ! grep -F "grep: warning:" pytest.out.txt )'
|
||||
- test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ( cd ../../.. && make clean >/dev/null 2>&1 )
|
||||
|
|
@ -555,10 +557,10 @@ stages:
|
|||
- cd "$CI_PROJECT_DIR"
|
||||
- *find_python
|
||||
- >
|
||||
"$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit_system.xml
|
||||
"$PYTHON" bin/tests/convert_trs_to_junit.py . > "$CI_PROJECT_DIR"/junit_system.xml
|
||||
- *git_clone_bind9-qa
|
||||
- >
|
||||
"$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit_system.xml --output "$CI_PROJECT_DIR"/junit.xml
|
||||
"$PYTHON" "$CI_PROJECT_DIR"/bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit_system.xml --output "$CI_PROJECT_DIR"/junit.xml
|
||||
- (exit $RET)
|
||||
after_script:
|
||||
- cat bin/tests/system/test-suite.log || true
|
||||
|
|
@ -584,11 +586,11 @@ stages:
|
|||
- RET=0
|
||||
- make -j${TEST_PARALLEL_JOBS:-1} -k unit V=1 || RET=$?
|
||||
- *find_python
|
||||
- >
|
||||
"$PYTHON" "$CI_PROJECT_DIR"/bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit_unit.xml
|
||||
- *git_clone_bind9-qa
|
||||
- >
|
||||
"$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit_unit.xml --output "$CI_PROJECT_DIR"/junit.xml
|
||||
"$PYTHON" "$CI_PROJECT_DIR"/bin/tests/convert_trs_to_junit.py . > "$CI_PROJECT_DIR"/junit_unit.xml
|
||||
- >
|
||||
"$PYTHON" "$CI_PROJECT_DIR"/bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit_unit.xml --output "$CI_PROJECT_DIR"/junit.xml
|
||||
- (exit $RET)
|
||||
- test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || make clean >/dev/null 2>&1
|
||||
artifacts:
|
||||
|
|
@ -620,7 +622,7 @@ stages:
|
|||
- make -j${BUILD_PARALLEL_JOBS:-1} V=1
|
||||
- *setup_interfaces
|
||||
- *git_clone_bind9-qa
|
||||
- cd bind9-qa/respdiff
|
||||
- cd "$CI_PROJECT_DIR"/bind9-qa/respdiff
|
||||
needs: []
|
||||
artifacts:
|
||||
paths:
|
||||
|
|
@ -693,7 +695,8 @@ ci-orphaned-anchors:
|
|||
<<: *quick_checks_job
|
||||
script:
|
||||
- *git_clone_bind9-qa
|
||||
- bind9-qa/ci-orphaned-anchors/check-orphaned-anchors-ci.py .gitlab-ci.yml
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/ci-orphaned-anchors/check_orphaned_anchors_ci.py .gitlab-ci.yml
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
||||
changes:
|
||||
|
|
@ -740,7 +743,7 @@ doctest:
|
|||
"$PYTEST" --noconftest --doctest-modules --junit-xml="$CI_PROJECT_DIR/junit_doctest.xml" || RET=1
|
||||
- *git_clone_bind9-qa
|
||||
- >
|
||||
"$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR/junit_doctest.xml" --output "$CI_PROJECT_DIR/junit.xml"
|
||||
"$PYTHON" "$CI_PROJECT_DIR"/bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR/junit_doctest.xml" --output "$CI_PROJECT_DIR/junit.xml"
|
||||
- (exit $RET)
|
||||
needs:
|
||||
- job: autoreconf
|
||||
|
|
@ -813,7 +816,8 @@ generate-stress-test-configs:
|
|||
<<: *quick_checks_job
|
||||
script:
|
||||
- *git_clone_bind9-qa
|
||||
- bind9-qa/stress/generate-stress-test-configs.py > stress-test-configs.yml
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/stress/generate_stress_test_configs.py > stress-test-configs.yml
|
||||
artifacts:
|
||||
paths:
|
||||
- stress-test-configs.yml
|
||||
|
|
@ -821,7 +825,9 @@ generate-stress-test-configs:
|
|||
generate-tsan-stress-test-configs:
|
||||
<<: *quick_checks_job
|
||||
script:
|
||||
- util/generate-tsan-stress-jobs.py > tsan-stress-test-configs.yml
|
||||
- *git_clone_bind9-qa
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/ci/generate_tsan_stress_jobs.py > tsan-stress-test-configs.yml
|
||||
artifacts:
|
||||
paths:
|
||||
- tsan-stress-test-configs.yml
|
||||
|
|
@ -1057,7 +1063,7 @@ cross-version-config-tests:
|
|||
"$PYTEST" --setup-only --junit-xml="$CI_PROJECT_DIR"/junit_pytest.xml -n "${TEST_PARALLEL_JOBS:-1}" || RET=1
|
||||
- *git_clone_bind9-qa
|
||||
- >
|
||||
"$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit_pytest.xml --output "$CI_PROJECT_DIR"/junit.xml
|
||||
"$PYTHON" "$CI_PROJECT_DIR"/bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit_pytest.xml --output "$CI_PROJECT_DIR"/junit.xml
|
||||
- (exit $RET)
|
||||
needs:
|
||||
- job: autoreconf
|
||||
|
|
@ -1873,7 +1879,8 @@ publish:
|
|||
variables:
|
||||
GIT_DEPTH: 1
|
||||
script:
|
||||
- bind9-qa/releng/printing_press_mr.py --document "${DOCUMENT}" --metadata bind9-qa/releng/metadata.json ${FORCE_CVE_IDS:+--force-cve-ids ${FORCE_CVE_IDS}} ${FORCE_SECURITY_RELEASES:+--force-security-releases ${FORCE_SECURITY_RELEASES}}
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/releng/printing_press_mr.py --document "${DOCUMENT}" --metadata "$CI_PROJECT_DIR"/bind9-qa/releng/metadata.json ${FORCE_CVE_IDS:+--force-cve-ids ${FORCE_CVE_IDS}} ${FORCE_SECURITY_RELEASES:+--force-security-releases ${FORCE_SECURITY_RELEASES}}
|
||||
artifacts:
|
||||
paths:
|
||||
- printing-press/
|
||||
|
|
@ -1931,7 +1938,8 @@ merge-tag:
|
|||
variables:
|
||||
GIT_DEPTH: 100
|
||||
script:
|
||||
- bind9-qa/releng/merge_tag.py --tag "$CI_COMMIT_TAG"
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/releng/merge_tag.py --tag "$CI_COMMIT_TAG"
|
||||
rules:
|
||||
- *rule_tag_open_source
|
||||
artifacts:
|
||||
|
|
@ -1965,7 +1973,8 @@ customer-git:branch:
|
|||
- *git_clone_bind9-qa
|
||||
script:
|
||||
- git checkout -b "$BRANCH" # ensure refs/heads/$BRANCH exists; GitLab clones with detached HEAD
|
||||
- bind9-qa/releng/push_to_customer_repository.py --branch "$BRANCH" --customer "$CUSTOMER" --force
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/releng/push_to_customer_repository.py --branch "$BRANCH" --customer "$CUSTOMER" --force
|
||||
|
||||
customer-git:tag:
|
||||
<<: *customer_git
|
||||
|
|
@ -1978,7 +1987,8 @@ customer-git:tag:
|
|||
- *git_clone_bind9-qa
|
||||
- git clone --depth 1 "https://token:${ISC_CUSTOMERS_WRITE_TOKEN}@gitlab.isc.org/isc-customers/isc-customer-settings.git"
|
||||
script:
|
||||
- bind9-qa/releng/push_to_customer_repository.py --tag "$CI_COMMIT_TAG" --entitlements isc-customer-settings/entitlements.yaml --force
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/releng/push_to_customer_repository.py --tag "$CI_COMMIT_TAG" --entitlements isc-customer-settings/entitlements.yaml --force
|
||||
|
||||
# Respdiff tests
|
||||
|
||||
|
|
@ -2051,7 +2061,7 @@ respdiff-third-party:
|
|||
- autoreconf -fi
|
||||
- *configure
|
||||
- make -j${BUILD_PARALLEL_JOBS:-1} V=1
|
||||
- cd bind9-qa/respdiff
|
||||
- cd "$CI_PROJECT_DIR"/bind9-qa/respdiff
|
||||
- bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}/version-under-test" "${CI_PROJECT_DIR}/bin/named/named"
|
||||
|
||||
respdiff:recent-named:
|
||||
|
|
@ -2203,14 +2213,16 @@ backports:
|
|||
script:
|
||||
# CI job token is not sufficient for push operations
|
||||
- git remote get-url origin | sed -e "s/gitlab-ci-token:$CI_JOB_TOKEN/oauth2:$BIND_TEAM_WRITE_TOKEN/" | xargs git remote set-url --push origin
|
||||
- bind9-qa/releng/backport_mr.py $CI_PROJECT_ID "$MERGE_REQUEST_ID"
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/releng/backport_mr.py $CI_PROJECT_ID "$MERGE_REQUEST_ID"
|
||||
|
||||
merged-metadata:
|
||||
<<: *post_merge
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "push" && ($CI_COMMIT_REF_NAME =~ /^bind-9.[0-9]+(-sub)?$/ || $CI_COMMIT_REF_NAME =~ /^v9.[0-9]+.[0-9]+-release$/ || $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH)'
|
||||
script:
|
||||
- bind9-qa/releng/after_merge.py "$CI_PROJECT_ID" "$MERGE_REQUEST_ID"
|
||||
- >
|
||||
"$CI_PROJECT_DIR"/bind9-qa/releng/after_merge.py "$CI_PROJECT_ID" "$MERGE_REQUEST_ID"
|
||||
|
||||
auto-rebase-trigger:
|
||||
stage: postmerge
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
include $(top_srcdir)/Makefile.top
|
||||
|
||||
EXTRA_DIST = convert-trs-to-junit.py
|
||||
EXTRA_DIST = convert_trs_to_junit.py
|
||||
|
||||
SUBDIRS = system
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ def junit_to_trs(junit_xml):
|
|||
testcases = root.findall(".//testcase")
|
||||
|
||||
if len(testcases) < 1:
|
||||
print(":test-result: ERROR convert-junit-to-trs.py")
|
||||
print(":test-result: ERROR convert_junit_to_trs.py")
|
||||
return 99
|
||||
|
||||
has_fail = False
|
||||
|
|
@ -146,7 +146,7 @@ else
|
|||
fi
|
||||
|
||||
# Run junit to trs converter script.
|
||||
./convert-junit-to-trs.py $junit_file >$trs_file
|
||||
./convert_junit_to_trs.py $junit_file >$trs_file
|
||||
estatus=$?
|
||||
|
||||
if test $enable_hard_errors = no && test $estatus -eq 99; then
|
||||
|
|
|
|||
|
|
@ -1,106 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
|
||||
#
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# See the COPYRIGHT file distributed with this work for additional
|
||||
# information regarding copyright ownership.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import gitlab
|
||||
|
||||
|
||||
def init_gitlab_project():
|
||||
if os.getenv("CI_SERVER_URL", None) is not None:
|
||||
url = os.getenv("CI_SERVER_URL")
|
||||
job_token = os.environ["CI_JOB_TOKEN"]
|
||||
gl = gitlab.Gitlab(url, job_token=job_token)
|
||||
else: # running locally on dev machine
|
||||
gl = gitlab.Gitlab.from_config()
|
||||
return gl.projects.get("isc-projects/bind9-shotgun-ci")
|
||||
|
||||
|
||||
def parse_parent_pipeline_id():
|
||||
if len(sys.argv) != 2:
|
||||
raise RuntimeError("usage: util/ci-wait-shotgun.py PIPELINE_ID")
|
||||
try:
|
||||
return int(sys.argv[1])
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("error: PIPELINE_ID must be a number") from exc
|
||||
|
||||
|
||||
def wait_until(callback, timeout=1800, retry=10):
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
if callback():
|
||||
return
|
||||
time.sleep(retry)
|
||||
raise RuntimeError(f"error: timed out after {timeout}s")
|
||||
|
||||
|
||||
def get_child_pipeline_id(project, pipeline_id):
|
||||
pipeline = project.pipelines.get(pipeline_id)
|
||||
|
||||
def pipeline_finished():
|
||||
pipeline.refresh()
|
||||
return pipeline.finished_at is not None
|
||||
|
||||
wait_until(pipeline_finished, timeout=6600)
|
||||
|
||||
bridges = pipeline.bridges.list()
|
||||
if len(bridges) != 1:
|
||||
raise RuntimeError("error: expected exactly one child pipeline")
|
||||
return bridges[0].downstream_pipeline["id"]
|
||||
|
||||
|
||||
def get_postproc_job(project, pipeline_id):
|
||||
pipeline = project.pipelines.get(pipeline_id)
|
||||
postproc_job = None
|
||||
|
||||
def job_finished():
|
||||
nonlocal postproc_job
|
||||
for job in pipeline.jobs.list(get_all=True):
|
||||
if job.name == "postproc":
|
||||
postproc_job = job
|
||||
if postproc_job is None:
|
||||
raise RuntimeError("error: failed to find 'postproc' job in child pipeline")
|
||||
return postproc_job.finished_at is not None
|
||||
|
||||
wait_until(job_finished)
|
||||
return postproc_job
|
||||
|
||||
|
||||
def evaluate_postproc_job(job):
|
||||
print(f"postproc job URL: {job.web_url}")
|
||||
index_url = (
|
||||
"https://isc-projects.gitlab-pages.isc.org/-/"
|
||||
f"bind9-shotgun-ci/-/jobs/{job.id}/artifacts/index.html"
|
||||
)
|
||||
if job.status in ["success", "failed"]:
|
||||
print(f"result for manual inspection: {index_url}")
|
||||
if job.status != "success":
|
||||
raise RuntimeError("error: 'postproc' job didn't succeed")
|
||||
|
||||
|
||||
def main():
|
||||
project = init_gitlab_project()
|
||||
parent_pipeline_id = parse_parent_pipeline_id()
|
||||
child_pipeline_id = get_child_pipeline_id(project, parent_pipeline_id)
|
||||
postproc_job = get_postproc_job(project, child_pipeline_id)
|
||||
evaluate_postproc_job(postproc_job)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except RuntimeError as err:
|
||||
print(err)
|
||||
sys.exit(1)
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
|
||||
#
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# See the COPYRIGHT file distributed with this work for additional
|
||||
# information regarding copyright ownership.
|
||||
|
||||
import yaml
|
||||
|
||||
NUMBER_OF_TESTS_PER_TSAN_JOB = 50
|
||||
|
||||
with open(".gitlab-ci.yml", encoding="utf-8") as gitlab_ci_yml:
|
||||
anchors = yaml.load(gitlab_ci_yml, Loader=yaml.Loader)
|
||||
|
||||
for tsan_job in "gcc:tsan", "clang:tsan":
|
||||
for test_type in "unit", "system":
|
||||
tsan_stress_test_job = anchors[f"{test_type}:{tsan_job}"]
|
||||
tsan_stress_test_job["stage"] = "test"
|
||||
tsan_stress_test_job["rules"] = [
|
||||
{"if": '$CI_PIPELINE_SOURCE == "parent_pipeline"'}
|
||||
]
|
||||
tsan_stress_test_job["parallel"] = NUMBER_OF_TESTS_PER_TSAN_JOB
|
||||
tsan_stress_test_job["needs"] = [
|
||||
{"pipeline": "$PARENT_PIPELINE_ID", "job": tsan_job}
|
||||
]
|
||||
|
||||
print(
|
||||
yaml.dump(
|
||||
{f"{test_type}:{tsan_job}:stress": tsan_stress_test_job},
|
||||
Dumper=yaml.Dumper,
|
||||
)
|
||||
)
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
|
||||
#
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# See the COPYRIGHT file distributed with this work for additional
|
||||
# information regarding copyright ownership.
|
||||
|
||||
import psutil
|
||||
|
||||
for pid in psutil.pids():
|
||||
try:
|
||||
environ = psutil.Process(pid).environ()
|
||||
if "PYTEST_CURRENT_TEST" in environ:
|
||||
name = psutil.Process(pid).name()
|
||||
print(
|
||||
f'pytest process {name}/{pid} running: {environ["PYTEST_CURRENT_TEST"]}'
|
||||
)
|
||||
except (psutil.AccessDenied, psutil.NoSuchProcess):
|
||||
pass
|
||||
|
|
@ -1,125 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
############################################################################
|
||||
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
|
||||
#
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# See the COPYRIGHT file distributed with this work for additional
|
||||
# information regarding copyright ownership.
|
||||
############################################################################
|
||||
|
||||
"""Parse the ThreadSanizer reports, unify them and put them into unique dirs."""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
from hashlib import sha256
|
||||
|
||||
|
||||
class State:
|
||||
"""Class that holds state of the TSAN parser."""
|
||||
|
||||
inside = False
|
||||
block = ""
|
||||
last_line = None
|
||||
|
||||
mutexes = {}
|
||||
m_index = 1
|
||||
threads = {}
|
||||
t_index = 1
|
||||
pointers = {}
|
||||
p_index = 1
|
||||
|
||||
def __init__(self):
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
"""Reset the object to initial state"""
|
||||
|
||||
self.inside = False
|
||||
self.block = ""
|
||||
|
||||
self.mutexes = {}
|
||||
self.threads = {}
|
||||
self.pointers = {}
|
||||
self.pointers["0x000000000000"] = 0
|
||||
|
||||
self.m_index = 1
|
||||
self.t_index = 1
|
||||
self.p_index = 1
|
||||
|
||||
|
||||
TOP = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
|
||||
OUT = os.path.join(TOP, "tsan")
|
||||
|
||||
if not os.path.isdir(OUT):
|
||||
os.mkdir(OUT)
|
||||
|
||||
# Regular Expressions
|
||||
MUTEX = re.compile(r"M\d+")
|
||||
THREAD = re.compile(r"T\d+")
|
||||
STACK = re.compile(r"\s\(\S+\+0x\S+\)")
|
||||
POINTER = re.compile(r"0x[0-9a-f]+")
|
||||
PID = re.compile(r"\(pid=\d+,?\)")
|
||||
TID = re.compile(r"tid=\d+,?\s*")
|
||||
WORKER = re.compile(r"\s+'(isc-worker|isc-net-)\d+'")
|
||||
PATH = re.compile(TOP + "/")
|
||||
|
||||
S = State()
|
||||
|
||||
with open(sys.argv[1], "r", encoding="utf-8") as f:
|
||||
for line in f.readlines():
|
||||
if line == "==================\n":
|
||||
if not S.inside:
|
||||
S.inside = True
|
||||
else:
|
||||
DNAME = sha256(S.last_line.encode("utf-8")).hexdigest()
|
||||
DNAME = os.path.join(OUT, DNAME)
|
||||
if not os.path.isdir(DNAME):
|
||||
os.mkdir(DNAME)
|
||||
FNAME = sha256(S.block.encode("utf-8")).hexdigest() + ".txt"
|
||||
FNAME = os.path.join(DNAME, FNAME)
|
||||
if not os.path.isfile(FNAME):
|
||||
with open(FNAME, "w", encoding="utf-8") as w:
|
||||
w.write(S.block)
|
||||
S.reset()
|
||||
else:
|
||||
for m in MUTEX.finditer(line):
|
||||
k = m.group()
|
||||
if k not in S.mutexes:
|
||||
S.mutexes[k] = S.m_index
|
||||
S.m_index += 1
|
||||
for m in THREAD.finditer(line):
|
||||
k = m.group()
|
||||
if k not in S.threads:
|
||||
S.threads[k] = S.t_index
|
||||
S.t_index += 1
|
||||
for m in POINTER.finditer(line):
|
||||
k = m.group()
|
||||
if k not in S.pointers:
|
||||
S.pointers[k] = S.p_index
|
||||
S.p_index += 1
|
||||
for k, v in S.mutexes.items():
|
||||
r = re.compile(k)
|
||||
line = r.sub("M{:04d}".format(v), line)
|
||||
for k, v in S.threads.items():
|
||||
r = re.compile(k)
|
||||
line = r.sub("T{:04d}".format(v), line)
|
||||
for k, v in S.pointers.items():
|
||||
r = re.compile(k)
|
||||
line = r.sub("0x{:012d}".format(v), line)
|
||||
|
||||
line = STACK.sub("", line)
|
||||
line = PID.sub("", line)
|
||||
line = TID.sub("", line)
|
||||
line = WORKER.sub("", line)
|
||||
line = PATH.sub("", line)
|
||||
|
||||
S.block += line
|
||||
S.last_line = line
|
||||
Loading…
Reference in a new issue