diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d25176252..2bada04e3e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -218,6 +218,18 @@ jobs: enable_npm: true - run: make install-goyacc check-generated-parser - run: make check-generated-promql-functions + check_generated_semconv: + name: Check generated semconv + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + persist-credentials: false + - name: Set up Weaver + uses: open-telemetry/weaver/setup-weaver@v0.20.0 + - name: Check semconv code is up-to-date + run: make check-semconv golangci: name: golangci-lint runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index f64f775993..8a76186a7f 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ npm_licenses.tar.bz2 /vendor /.build +/.bin /go.work.sum /**/node_modules diff --git a/Makefile b/Makefile index 8bc4a3dcaa..e14b571603 100644 --- a/Makefile +++ b/Makefile @@ -225,3 +225,16 @@ bump-go-version: generate-fuzzing-seed-corpus: @echo ">> Generating fuzzing seed corpus" @$(GO) generate -tags fuzzing ./util/fuzzing/corpus_gen + +.PHONY: generate-semconv +generate-semconv: + @echo ">> generating semconv code" + @./build/semconv/generate.sh + +.PHONY: check-semconv +check-semconv: generate-semconv + @echo ">> checking semconv code is up-to-date" + @if ! git diff --exit-code -- '*/semconv/metrics.go' '*/semconv/README.md'; then \ + echo "Generated semconv code is out of date. Run 'make generate-semconv' and commit the changes."; \ + exit 1; \ + fi diff --git a/build/semconv/generate.sh b/build/semconv/generate.sh new file mode 100755 index 0000000000..ee41b052d6 --- /dev/null +++ b/build/semconv/generate.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Copyright The Prometheus Authors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script generates Go code from semantic convention registries. +# It finds all registry.yaml files in semconv/ directories and generates +# metrics.go files in the same directories. +# +# Usage: +# ./build/semconv/generate.sh +# +# Requirements: +# - gofmt must be available +# - curl or wget for downloading weaver (if not installed) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +TEMPLATES="${SCRIPT_DIR}/templates" + +# Weaver version to use - update this when upgrading +WEAVER_VERSION="v0.20.0" + +# Local bin directory for downloaded tools +LOCAL_BIN="${REPO_ROOT}/.bin" +WEAVER_BIN="${LOCAL_BIN}/weaver-${WEAVER_VERSION}" + +# Detect OS and architecture for downloading weaver +detect_platform() { + local os arch + + case "$(uname -s)" in + Linux*) os="unknown-linux-gnu" ;; + Darwin*) os="apple-darwin" ;; + *) echo "Unsupported OS: $(uname -s)"; exit 1 ;; + esac + + case "$(uname -m)" in + x86_64) arch="x86_64" ;; + aarch64) arch="aarch64" ;; + arm64) arch="aarch64" ;; + *) echo "Unsupported architecture: $(uname -m)"; exit 1 ;; + esac + + echo "${arch}-${os}" +} + +# Download weaver if not present +install_weaver() { + local platform="$1" + local tarball="weaver-${platform}.tar.xz" + local url="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${tarball}" + + echo ">> Installing weaver ${WEAVER_VERSION} for ${platform}" + mkdir -p "${LOCAL_BIN}" + + # Download using curl or wget + if command -v curl &> /dev/null; then + if ! curl -sSfL "${url}" -o "${LOCAL_BIN}/${tarball}"; then + echo "Error: Failed to download weaver from ${url}" + exit 1 + fi + elif command -v wget &> /dev/null; then + if ! wget -q "${url}" -O "${LOCAL_BIN}/${tarball}"; then + echo "Error: Failed to download weaver from ${url}" + exit 1 + fi + else + echo "Error: Neither curl nor wget found. Please install one of them." + exit 1 + fi + + # Extract the binary (tar.xz format) + # The archive contains a directory like weaver-aarch64-apple-darwin/weaver + if ! tar -xJf "${LOCAL_BIN}/${tarball}" -C "${LOCAL_BIN}"; then + echo "Error: Failed to extract weaver archive" + rm -f "${LOCAL_BIN}/${tarball}" + exit 1 + fi + mv "${LOCAL_BIN}/weaver-${platform}/weaver" "${WEAVER_BIN}" + chmod +x "${WEAVER_BIN}" + + # Cleanup tarball and extracted directory + rm -f "${LOCAL_BIN}/${tarball}" + rm -rf "${LOCAL_BIN}/weaver-${platform}" + + echo ">> Installed weaver to ${WEAVER_BIN}" +} + +# Get the weaver binary path, installing if necessary +get_weaver() { + # First check if the pinned version is already downloaded + if [ -x "${WEAVER_BIN}" ]; then + echo "${WEAVER_BIN}" + return + fi + + # Check if weaver is in PATH and matches our version + if command -v weaver &> /dev/null; then + local installed_version + installed_version=$(weaver --version 2>/dev/null | head -1 | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' || echo "") + if [ "${installed_version}" = "${WEAVER_VERSION}" ]; then + echo "weaver" + return + fi + echo ">> System weaver version (${installed_version}) differs from required (${WEAVER_VERSION})" >&2 + fi + + # Download and install + local platform + platform=$(detect_platform) + install_weaver "${platform}" >&2 + echo "${WEAVER_BIN}" +} + +# Get weaver binary +WEAVER=$(get_weaver) +echo ">> Using weaver: ${WEAVER}" + +# Check if gofmt is installed +if ! command -v gofmt &> /dev/null; then + echo "Error: gofmt is not installed." + echo "Install Go from: https://go.dev/dl/" + exit 1 +fi + +# Find all registries (directories containing registry.yaml under semconv/) +REGISTRIES=$(find "${REPO_ROOT}" -path '*/semconv/registry.yaml' -type f 2>/dev/null || true) + +if [ -z "${REGISTRIES}" ]; then + echo "No semconv registries found." + echo "Registries should be placed in */semconv/registry.yaml" + exit 0 +fi + +echo "Found registries:" +echo "${REGISTRIES}" | while read -r registry; do + echo " - ${registry}" +done +echo "" + +# Generate code for each registry +for registry in ${REGISTRIES}; do + dir=$(dirname "${registry}") + echo ">> Generating ${dir}/metrics.go and ${dir}/README.md" + + "${WEAVER}" registry generate \ + --registry "${dir}" \ + --templates "${TEMPLATES}" \ + go "${dir}" \ + --skip-policies +done + +# Format all generated files +echo "" +echo ">> Formatting generated files" +find "${REPO_ROOT}" -path '*/semconv/metrics.go' -type f -exec gofmt -w {} \; + +echo "" +echo "Done! Generated files:" +find "${REPO_ROOT}" -path '*/semconv/metrics.go' -type f | while read -r file; do + echo " - ${file}" +done +find "${REPO_ROOT}" -path '*/semconv/README.md' -type f | while read -r file; do + echo " - ${file}" +done diff --git a/build/semconv/templates/go/helpers.j2 b/build/semconv/templates/go/helpers.j2 new file mode 100644 index 0000000000..227d6dd79a --- /dev/null +++ b/build/semconv/templates/go/helpers.j2 @@ -0,0 +1,104 @@ +{# +Copyright The Prometheus Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} +{%- macro repl(text) -%} +{#- Copied from semconvgen: https://github.com/open-telemetry/opentelemetry-go-build-tools/blob/3e69152c51c56213b65c0fc6e5954293b522103c/semconvgen/generator.go#L419-L426 -#} +{{ text | replace("RedisDatabase", "RedisDB") | replace("IPTCP", "TCP") | replace("IPUDP", "UDP") | replace("Lineno", "LineNumber") }} +{%- endmacro -%} + +{%- macro smart_title_case(text) -%} +{%- for i in range(0, text | length) -%} + {%- if i == 0 or text[i-1] in ['.', '_'] -%} + {{ text[i] | upper }} + {%- elif not text[i] in ['.', '_'] -%} + {{ text[i] }} + {%- endif -%} +{%- endfor -%} +{%- endmacro -%} + +{%- macro to_go_name(fqn="", pkg="") -%} +{%- if pkg != "" and fqn != pkg -%} +{%- set n = pkg | length -%} +{%- if pkg == fqn[:n] -%} +{%- set fqn = fqn[n:] -%} +{%- if fqn[0] == "." or fqn[0] == "." -%} +{%- set fqn = fqn[1:] -%} +{%- endif -%} +{%- endif -%} +{%- endif -%} +{{ repl(smart_title_case(fqn | replace(" ", "") | replace("_", ".") | acronym)) }} +{%- endmacro -%} + +{%- macro lower_first(line) -%} +{%- if line is string and line | length > 1 -%} +{%- if line[0] is upper and line[1] is upper -%} +{#- Assume an acronym -#} +{{ line }} +{%- else -%} +{{ line[0]|lower }}{{ line[1:] }} +{%- endif -%} +{%- elif line is not none -%} +{{ line }} +{%- endif -%} +{%- endmacro -%} + +{%- macro first_word(line, delim=" ") -%} +{%- for c in line -%} +{%- if c == delim -%} +{{ line[:loop.index0] }} +{%- set line = "" -%} +{%- endif -%} +{%- endfor -%} +{%- endmacro -%} + +{%- macro prefix_brief(brief, prefix="") -%} +{%- set norms = [ + "MUST", "REQUIRED", "SHALL", + "SHOULD", "RECOMMENDED", + "MAY", "OPTIONAL" +] -%} +{%- set brief = brief | trim() -%} +{%- if first_word(brief) is in norms -%} +It {{ brief }}. +{%- else -%} +{{ prefix }} {% if brief[:2] == "A " or brief[:3] == "An " or brief[:4] == "The " -%} + {{ lower_first(brief) | trim(".") }}. +{%- else -%} + the {{ lower_first(brief) | trim(".") }}. +{%- endif -%} +{%- endif -%} +{%- endmacro -%} + +{%- macro it_reps(brief) -%} +{{ prefix_brief(brief, "It represents") }} +{%- endmacro -%} + +{%- macro metric_typedoc(metric, pkg="") -%} +{%- set name = to_go_name(metric.metric_name, pkg=pkg) -%} +{%- set brief = metric.brief | default("") | trim | trim(".") -%} +{%- if not brief -%} +{{ name }} records the {{ metric.metric_name }} metric. +{%- elif brief[:2] == "A " or brief[:3] == "An " or brief[:4] == "The " -%} +{{ name }} records {{ lower_first(brief) }}. +{%- else -%} +{{ name }} records the {{ lower_first(brief) }}. +{%- endif %} +{%- endmacro -%} + +{%- macro member_type(member) %} +{%- if member.value is string %}string{%- endif %} +{%- if member.value is boolean %}bool{%- endif %} +{%- if member.value is int %}int64{%- endif %} +{%- if member.value is float %}float64{%- endif %} +{%- endmacro %} diff --git a/build/semconv/templates/go/metrics.go.j2 b/build/semconv/templates/go/metrics.go.j2 new file mode 100644 index 0000000000..c261974af8 --- /dev/null +++ b/build/semconv/templates/go/metrics.go.j2 @@ -0,0 +1,148 @@ +{% import 'helpers.j2' as h -%} +{# Flatten all metrics from all groups into a single list #} +{%- set all_metrics = ctx | map(attribute="metrics") | flatten | list -%} +{# Extract all unique attributes from all metrics #} +{%- set all_attrs = all_metrics | selectattr("attributes") | map(attribute="attributes") | flatten | unique(attribute="name") | sort(attribute="name") -%} +// Code generated from semantic convention specification. DO NOT EDIT. + +// Package metrics provides Prometheus instrumentation types for metrics +// defined in this semantic convention registry. +package metrics + +import ( +{%- if all_attrs | selectattr("type","!=","string") | list | length > 0 %} + "fmt" +{%- endif %} + "github.com/prometheus/client_golang/prometheus" +) + +// Attribute is an interface for metric label attributes. +type Attribute interface { + ID() string + Value() string +} +{%- for attr in all_attrs %} +{%- set name = h.to_go_name(attr.name, "") %} + +{%- if attr.type.members is not defined %} +type {{ name }}Attr {{ attr.type }} +{%- else %} +type {{ name }}Attr {{ h.member_type(attr.type.members[0]) }} + +var ( +{%- for m in attr.type.members %} + {%- set m_name = name ~ h.to_go_name(m.id, "") -%} + {% if attr.type.members[0].value is string -%} + {%- set m_value = '"' + m.value + '"' -%} + {%- else -%} + {%- set m_value = m.value -%} + {%- endif -%} + {%- if m.brief is defined %} + {%- set m_brief = m.brief -%} + {%- else %} + {%- set m_brief = "standardized value " + m_value + ' of ' + name + 'Attr.' -%} + {%- endif %} +{{ h.prefix_brief(m_brief, m_name ~ " is ") | comment(format="go_1tab") }} + {{ m_name }} {{ name }}Attr = {{ m_value }} +{%- endfor %} +) +{%- endif %} + +func (a {{ name }}Attr) ID() string { + return "{{ attr.name }}" +} + +func (a {{ name }}Attr) Value() string { + {% if attr.type == "string" -%} + return string(a) + {%- elif attr.type == "int" -%} + return fmt.Sprintf("%d", a) + {%- else -%} + return fmt.Sprintf("%v", a) + {%- endif %} +} +{%- endfor %} + +{% macro for_each_attr(attrs) %} +{%- for raw in attrs -%} +{%- set attr = namespace(raw) -%} +{%- set attr.id = attr.name -%} +{%- set attr.namespace = attr.name | attribute_namespace -%} +{%- set attr.name = attr.name | attribute_id | pascal_case -%} +{%- set attr.arg = attr.name | replace("Type", "Kind") | camel_case -%} +{%- set attr.fullname = attr.id | pascal_case -%} +{%- set attr.pkg = "" -%} +{%- set attr.type = attr.fullname+"Attr" -%} +{%- set attr.ref = attr.pkg+attr.type -%} +{%- set attr.getter = attr.id|pascal_case -%} +{%- set attr.field = "Attr"+attr.fullname -%} +{{ caller(attr) }} +{%- endfor -%} +{% endmacro %} +{%- for metric in all_metrics %} +{%- set metric_name = h.to_go_name(metric.metric_name, "") %} +{%- set metric_inst = (metric.instrument | default("gauge")) | map_text("go_instrument_type") %} +{%- set metric_attr = metric.attributes | default([]) | sort -%} + +{{ h.metric_typedoc(metric, "") | comment | trim }} +type {{ metric_name }} struct { + *prometheus.{{ metric_inst }}Vec +} + +{{ ["New" ~ metric_name ~ " returns a new " ~ metric_name ~ " instrument."] | comment }} +func New{{ metric_name }}() {{ metric_name }} { +{%- if metric_attr | length > 0 %} + labels := []string{ + {%- call(attr) for_each_attr(metric_attr) %} + "{{ attr.id }}", + {%- endcall %} + } +{%- else %} + labels := []string{} +{%- endif %} + return {{ metric_name }}{ + {{ metric_inst }}Vec: prometheus.New{{ metric_inst }}Vec(prometheus.{{ metric_inst }}Opts{ + Name: "{{ metric.metric_name }}", + Help: "{{ metric.brief | default("") | trim }}", + }, labels), + } +} + +type {{ metric_name }}Attr interface { + Attribute + impl{{ metric_name }}() +} +{%- if metric_attr | length > 0 %} +{% call(attr) for_each_attr(metric_attr) %} +func (a {{ attr.type }}) impl{{ metric_name }}() {} +{%- endcall %} +{%- endif %} + +func (m {{ metric_name }}) With( +{%- call(attr) for_each_attr(metric_attr|required) %} + {{ attr.arg }} {{ attr.type }}, +{%- endcall %} + extra ...{{ metric_name }}Attr, +) prometheus.{{ metric_inst | replace("Histogram", "Observer") }} { +{%- if metric_attr | length > 0 %} + labels := prometheus.Labels{ + {%- call(attr) for_each_attr(metric_attr|required) %} + "{{ attr.id }}": {{ attr.arg }}.Value(), + {%- endcall %} + {%- call(attr) for_each_attr(metric_attr|not_required) %} + "{{ attr.id }}": "", + {%- endcall %} + } +{%- else %} + labels := prometheus.Labels{} +{%- endif %} + for _, v := range extra { + labels[v.ID()] = v.Value() + } + return m.{{ metric_inst }}Vec.With(labels) +} +{%- if not loop.last %} + +{% endif %} +{%- endfor %} + diff --git a/build/semconv/templates/go/metrics.md.j2 b/build/semconv/templates/go/metrics.md.j2 new file mode 100644 index 0000000000..65ad714f51 --- /dev/null +++ b/build/semconv/templates/go/metrics.md.j2 @@ -0,0 +1,57 @@ +{# +Copyright The Prometheus Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} +{%- set all_metrics = ctx | map(attribute="metrics") | flatten | list -%} + + +# Metrics + +This document describes the metrics defined in this semantic convention registry. + +| Metric | Type | Unit | Description | +|--------|------|------|-------------| +{% for metric in all_metrics | sort(attribute="metric_name") -%} +{% set deprecated_badge = "**DEPRECATED** " if metric.deprecated else "" -%} +| `{{ metric.metric_name }}` | {{ metric.instrument | default("gauge") }} | {{ metric.unit | default("-") }} | {{ deprecated_badge }}{{ metric.brief | default("") | trim }} | +{% endfor %} + +## Metric Details +{% for metric in all_metrics | sort(attribute="metric_name") %} + +### `{{ metric.metric_name }}` +{%- if metric.deprecated %} + +> **Deprecated:** {{ metric.deprecated }} +{%- endif %} + +{{ metric.brief | default("No description available.") }} + +- **Type:** {{ metric.instrument | default("gauge") }} +- **Unit:** {{ metric.unit | default("unspecified") }} +- **Stability:** {{ metric.stability | default("development") }} +{%- if metric.note %} + +**Note:** {{ metric.note }} +{%- endif %} +{%- if metric.attributes %} + +#### Attributes + +| Attribute | Type | Description | Examples | +|-----------|------|-------------|----------| +{% for attr in metric.attributes | sort(attribute="name") -%} +| `{{ attr.name }}` | {{ attr.type }} | {{ attr.brief | default("") | trim }} | {{ attr.examples | default([]) | join(", ") }} | +{% endfor -%} +{%- endif %} +{% endfor %}