diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fe75a6fa..75a8ff62d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: - uses: actions/checkout@v4 - name: make run: | - sudo apt-get update && sudo apt-get install libc6-dev-i386 gcc-multilib g++-multilib + sudo apt-get update && sudo apt-get install libc6-dev-i386 gcc-multilib make REDIS_CFLAGS='-Werror' 32bit build-libc-malloc: @@ -79,7 +79,7 @@ jobs: - uses: actions/checkout@v4 - name: make run: | - dnf -y install which gcc gcc-c++ make + dnf -y install which gcc make make REDIS_CFLAGS='-Werror' build-old-chain-jemalloc: @@ -96,7 +96,6 @@ jobs: apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5 apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32 apt-get update - apt-get install -y make gcc-4.8 g++-4.8 + apt-get install -y make gcc-4.8 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100 - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100 make CC=gcc REDIS_CFLAGS='-Werror' diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 82656ac31..5108ec907 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -18,7 +18,7 @@ jobs: make lcov - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./src/redis.info diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 0237c8739..f5d37ae5c 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -14,9 +14,11 @@ jobs: - uses: actions/checkout@main - name: Download and extract the Coverity Build Tool run: | - wget -q https://scan.coverity.com/download/cxx/linux64 --post-data "token=${{ secrets.COVERITY_SCAN_TOKEN }}&project=redis-unstable" -O cov-analysis-linux64.tar.gz + wget -q https://scan.coverity.com/download/cxx/linux64 --post-data "token=${COVERITY_SCAN_TOKEN}&project=redis-unstable" -O cov-analysis-linux64.tar.gz mkdir cov-analysis-linux64 tar xzf cov-analysis-linux64.tar.gz --strip 1 -C cov-analysis-linux64 + env: + COVERITY_SCAN_TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} - name: Install Redis dependencies run: sudo apt install -y gcc tcl8.6 tclx procps libssl-dev - name: Build with cov-build @@ -26,7 +28,10 @@ jobs: tar czvf cov-int.tgz cov-int curl \ --form project=redis-unstable \ - --form email=${{ secrets.COVERITY_SCAN_EMAIL }} \ - --form token=${{ secrets.COVERITY_SCAN_TOKEN }} \ + --form email="${COVERITY_SCAN_EMAIL}" \ + --form token="${COVERITY_SCAN_TOKEN}" \ --form file=@cov-int.tgz \ https://scan.coverity.com/builds + env: + COVERITY_SCAN_EMAIL: ${{ secrets.COVERITY_SCAN_EMAIL }} + COVERITY_SCAN_TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index b70f98618..36edb7529 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -52,7 +52,7 @@ jobs: repository: ${{ env.GITHUB_REPOSITORY }} ref: ${{ env.GITHUB_HEAD_REF }} - name: make - run: make REDIS_CFLAGS='-Werror -DREDIS_TEST' + run: make REDIS_CFLAGS='-Werror -DREDIS_TEST -DDEBUG_ASSERTIONS' - name: testprep run: sudo apt-get install tcl8.6 tclx - name: test @@ -240,7 +240,7 @@ jobs: ref: ${{ env.GITHUB_HEAD_REF }} - name: make run: | - apt-get update && apt-get install -y make gcc g++ + apt-get update && apt-get install -y make gcc make CC=gcc REDIS_CFLAGS='-Werror -DREDIS_TEST -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3' - name: testprep run: sudo apt-get install -y tcl8.6 tclx procps @@ -347,7 +347,7 @@ jobs: ref: ${{ env.GITHUB_HEAD_REF }} - name: make run: | - sudo apt-get update && sudo apt-get install libc6-dev-i386 g++ gcc-multilib g++-multilib + sudo apt-get update && sudo apt-get install libc6-dev-i386 gcc-multilib make 32bit REDIS_CFLAGS='-Werror -DREDIS_TEST' make -C tests/modules 32bit # the script below doesn't have an argument, we must build manually ahead of time - name: testprep @@ -580,7 +580,7 @@ jobs: - name: testprep run: | sudo apt-get update - sudo apt-get install tcl8.6 tclx valgrind g++ -y + sudo apt-get install tcl8.6 tclx valgrind -y - name: test if: true && !contains(github.event.inputs.skiptests, 'redis') # Note that valgrind's overhead doesn't pair well with io-threads so we @@ -645,7 +645,7 @@ jobs: - name: testprep run: | sudo apt-get update - sudo apt-get install tcl8.6 tclx valgrind g++ -y + sudo apt-get install tcl8.6 tclx valgrind -y - name: test if: true && !contains(github.event.inputs.skiptests, 'redis') run: ./runtest --valgrind --tags -iothreads --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} @@ -878,7 +878,7 @@ jobs: ref: ${{ env.GITHUB_HEAD_REF }} - name: make run: | - dnf -y install which gcc make g++ + dnf -y install which gcc make make REDIS_CFLAGS='-Werror' - name: testprep run: | @@ -917,7 +917,7 @@ jobs: ref: ${{ env.GITHUB_HEAD_REF }} - name: make run: | - dnf -y install which gcc make openssl-devel openssl g++ + dnf -y install which gcc make openssl-devel openssl make BUILD_TLS=module REDIS_CFLAGS='-Werror' - name: testprep run: | @@ -960,7 +960,7 @@ jobs: ref: ${{ env.GITHUB_HEAD_REF }} - name: make run: | - dnf -y install which gcc make openssl-devel openssl g++ + dnf -y install which gcc make openssl-devel openssl make BUILD_TLS=module REDIS_CFLAGS='-Werror' - name: testprep run: | @@ -1093,9 +1093,6 @@ jobs: (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) && !contains(github.event.inputs.skipjobs, 'freebsd') timeout-minutes: 360 - env: - CC: clang - CXX: clang++ steps: - name: prep if: github.event_name == 'workflow_dispatch' @@ -1224,7 +1221,7 @@ jobs: if: true && !contains(github.event.inputs.skiptests, 'cluster') run: ./runtest-cluster --log-req-res --dont-clean --force-resp3 ${{github.event.inputs.cluster_test_args}} - name: Install Python dependencies - uses: py-actions/py-dependency-install@v4 + uses: py-actions/py-dependency-install@30aa0023464ed4b5b116bd9fbdab87acf01a484e # v4.1.0 with: path: "./utils/req-res-validator/requirements.txt" - name: validator @@ -1260,9 +1257,8 @@ jobs: apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5 apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32 apt-get update - apt-get install -y make gcc-4.8 g++-4.8 + apt-get install -y make gcc-4.8 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100 - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100 make CC=gcc REDIS_CFLAGS='-Werror' - name: testprep run: apt-get install -y tcl tcltls tclx @@ -1306,10 +1302,9 @@ jobs: apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5 apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32 apt-get update - apt-get install -y make gcc-4.8 g++-4.8 openssl libssl-dev + apt-get install -y make gcc-4.8 openssl libssl-dev update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100 - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100 - make CC=gcc CXX=g++ BUILD_TLS=module REDIS_CFLAGS='-Werror' + make CC=gcc BUILD_TLS=module REDIS_CFLAGS='-Werror' - name: testprep run: | apt-get install -y tcl tcltls tclx @@ -1357,9 +1352,8 @@ jobs: apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5 apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32 apt-get update - apt-get install -y make gcc-4.8 g++-4.8 openssl libssl-dev + apt-get install -y make gcc-4.8 openssl libssl-dev update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100 - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100 make BUILD_TLS=module CC=gcc REDIS_CFLAGS='-Werror' - name: testprep run: | diff --git a/.github/workflows/post-release-automation.yml b/.github/workflows/post-release-automation.yml index b7bf20235..94d9cc52a 100644 --- a/.github/workflows/post-release-automation.yml +++ b/.github/workflows/post-release-automation.yml @@ -5,17 +5,15 @@ on: types: [published] jobs: - automate-release-scripts: - # Only run for the main redis/redis repository (not forks) - # Note: Only users with write access can publish releases, providing implicit authorization + extract-release-info: if: github.repository == 'redis/redis' runs-on: ubuntu-latest - + outputs: + tag_name: ${{ steps.release-info.outputs.tag_name }} + release_type: ${{ steps.release-info.outputs.release_type }} steps: - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for git archive + uses: actions/checkout@v5 - name: Extract and validate release information id: release-info @@ -23,15 +21,12 @@ jobs: TAG_NAME: ${{ github.event.release.tag_name }} GH_TOKEN: ${{ github.token }} run: | - # Extract tag name from the release event (via env var to prevent injection) echo "tag_name=${TAG_NAME}" >> $GITHUB_OUTPUT echo "Release tag: ${TAG_NAME}" - # Get the latest release tag LATEST_TAG=$(gh release view --json tagName --jq '.tagName') - echo "Latest release tag from gh cli: ${LATEST_TAG}" + echo "Latest release tag(from gh release): ${LATEST_TAG}" - # Determine release type by comparing with latest release if [[ "${TAG_NAME}" == "${LATEST_TAG}" ]]; then echo "release_type=latest" >> $GITHUB_OUTPUT echo "Detected latest release: ${TAG_NAME}" @@ -40,116 +35,131 @@ jobs: echo "Detected non-latest release: ${TAG_NAME} (latest is ${LATEST_TAG})" fi - - name: Set up environment variables - run: | - echo "RELEASE_TAG=${{ steps.release-info.outputs.tag_name }}" >> $GITHUB_ENV - echo "RELEASE_TYPE=${{ steps.release-info.outputs.release_type }}" >> $GITHUB_ENV - echo "Environment variables set:" - echo " RELEASE_TAG: ${{ steps.release-info.outputs.tag_name }}" - echo " RELEASE_TYPE: ${{ steps.release-info.outputs.release_type }}" - + create-tarball: + needs: extract-release-info + runs-on: ubuntu-latest + env: + TAG_NAME: ${{ needs.extract-release-info.outputs.tag_name }} + outputs: + sha256: ${{ steps.checksum.outputs.sha256 }} + size_mb: ${{ steps.size.outputs.size_mb }} + size_warning: ${{ steps.size.outputs.size_warning }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + ref: ${{ env.TAG_NAME }} + fetch-depth: 0 + - name: Create tarball - id: create-tarball + run: ./utils/releasetools/01_create_tarball.sh "$TAG_NAME" + + - name: Verify tarball size + id: size run: | - echo "Creating tarball for version ${RELEASE_TAG}..." - # TODO: Implement tarball creation using utils/releasetools/01_create_tarball.sh - # ./utils/releasetools/01_create_tarball.sh ${RELEASE_TAG} - - # Placeholder: Verify tarball was created - # TARBALL_PATH="/tmp/redis-${RELEASE_TAG}.tar.gz" - # if [ ! -f "${TARBALL_PATH}" ]; then - # echo "Error: Tarball not found at ${TARBALL_PATH}" - # exit 1 - # fi - # echo "tarball_path=${TARBALL_PATH}" >> $GITHUB_OUTPUT - - echo "✓ Tarball creation step (placeholder)" - - - name: Upload tarball - id: upload-tarball - run: | - echo "Uploading tarball for version ${RELEASE_TAG}..." - # TODO: Implement tarball upload - # This will require: - # - SSH credentials/keys for upload to download.redis.io - # - Adaptation of utils/releasetools/02_upload_tarball.sh for CI environment - - echo "✓ Tarball upload step (placeholder)" - - - name: Test release tarball - id: test-release - run: | - echo "Testing release tarball for version ${RELEASE_TAG}..." - # TODO: Implement release testing using utils/releasetools/03_test_release.sh - # This will: - # - Download the uploaded tarball - # - Extract and build Redis - - echo "✓ Release testing step (placeholder)" - - - name: Update release hashes - id: update-hashes - run: | - echo "Updating release hashes for version ${RELEASE_TAG}..." - # TODO: Implement hash update using utils/releasetools/04_release_hash.sh - # This will require: - # - Access to redis-hashes repository - # - Git credentials for committing and pushing - - echo "✓ Release hashes update step (placeholder)" - - - name: Approval gate for latest releases - if: steps.release-info.outputs.release_type == 'latest' - run: | - echo "Latest release detected. Manual approval required for production deployment." - # TODO: Implement approval workflow - # This could use GitHub Environments with required reviewers - # or a manual approval step - - echo "✓ Approval gate (placeholder)" - - - name: Update stable symlink (latest releases only) - if: steps.release-info.outputs.release_type == 'latest' - id: update-stable - run: | - echo "This is a latest release. Updating stable symlink after approval." - # TODO: Implement stable symlink update - # This step should only run for latest releases (not non-latest) - # It will update the redis-stable symlink on download.redis.io - # This is part of the upload script (02_upload_tarball.sh) - - echo "✓ Stable symlink update step (placeholder)" - - - name: Summary - if: always() - run: | - echo "## Post-Release Automation Summary" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "- **Release Tag:** ${{ steps.release-info.outputs.tag_name }}" >> $GITHUB_STEP_SUMMARY - echo "- **Release Type:** ${{ steps.release-info.outputs.release_type }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Steps Status" >> $GITHUB_STEP_SUMMARY - echo "- Create tarball: ${{ steps.create-tarball.outcome }}" >> $GITHUB_STEP_SUMMARY - echo "- Upload tarball: ${{ steps.upload-tarball.outcome }}" >> $GITHUB_STEP_SUMMARY - echo "- Test release: ${{ steps.test-release.outcome }}" >> $GITHUB_STEP_SUMMARY - echo "- Update hashes: ${{ steps.update-hashes.outcome }}" >> $GITHUB_STEP_SUMMARY - if [[ "${{ steps.release-info.outputs.release_type }}" == "latest" ]]; then - echo "- Update stable symlink: ${{ steps.update-stable.outcome }}" >> $GITHUB_STEP_SUMMARY + TARBALL="/tmp/redis-${TAG_NAME}.tar.gz" + SIZE_MB=$(du -m "$TARBALL" | cut -f1) + echo "Tarball size: ${SIZE_MB} MB" + echo "size_mb=${SIZE_MB}" >> $GITHUB_OUTPUT + if [ "$SIZE_MB" -lt 3 ] || [ "$SIZE_MB" -gt 5 ]; then + echo "::warning::Tarball size ${SIZE_MB} MB is outside expected range (3-5 MB)" + echo "size_warning=true" >> $GITHUB_OUTPUT + else + echo "size_warning=false" >> $GITHUB_OUTPUT fi - - name: Send Slack notification - if: always() + - name: Calculate SHA256 checksum + id: checksum run: | - echo "Sending Slack notification for release ${RELEASE_TAG}..." - # TODO: Implement Slack notification - # This will require: - # - Slack webhook URL or bot token (stored in secrets) - # - Determine appropriate channel (e.g., #releases, #redis-releases) - # - Craft message with release information and workflow status - # Example using webhook: - # curl -X POST -H 'Content-type: application/json' \ - # --data '{"channel":"#releases","text":"Release ${RELEASE_TAG} automation completed"}' \ - # ${{ secrets.SLACK_WEBHOOK_URL }} + TARBALL="/tmp/redis-${TAG_NAME}.tar.gz" + SHA256=$(shasum -a 256 "$TARBALL" | cut -d' ' -f1) + echo "SHA256: $SHA256" + echo "sha256=$SHA256" >> $GITHUB_OUTPUT - echo "✓ Slack notification step (placeholder)" + - name: Upload tarball as artifact + uses: actions/upload-artifact@v6 + with: + name: redis-${{ env.TAG_NAME }}-tarball + path: /tmp/redis-${{ env.TAG_NAME }}.tar.gz + compression-level: 0 + # approval-gate: + # needs: [extract-release-info, create-tarball] + # if: needs.extract-release-info.outputs.release_type == 'latest' + # runs-on: ubuntu-latest + # steps: + # - name: Approval gate + # run: | + # echo "Latest release detected. Manual approval required for production deployment." + # # TODO: Implement approval workflow + # # This could use GitHub Environments with required reviewers + # # or a manual approval step + + # upload-tarball: + # needs: [extract-release-info, create-tarball, approval-gate] + # if: always() && !cancelled() && needs.create-tarball.result == 'success' && (needs.approval-gate.result == 'success' || needs.approval-gate.result == 'skipped') + # runs-on: ubuntu-latest + # steps: + # - name: Upload tarball + # run: | + # echo "TODO: Implement tarball upload" + # # This will require: + # # - SSH credentials/keys for upload to download.redis.io + # # - Adaptation of utils/releasetools/02_upload_tarball.sh for CI environment + + # test-release-tarball: + # needs: upload-tarball + # runs-on: ubuntu-latest + # steps: + # - name: Test release tarball + # run: | + # echo "TODO: Implement release testing using utils/releasetools/03_test_release.sh" + # # This will: + # # - Download the uploaded tarball + # # - Extract and build Redis + + # update-release-hashes: + # needs: test-release-tarball + # runs-on: ubuntu-latest + # steps: + # - name: Update release hashes + # run: | + # echo "TODO: Implement hash update using utils/releasetools/04_release_hash.sh" + # # This will require: + # # - Access to redis-hashes repository + # # - Git credentials for committing and pushing + + summary-and-notify: + needs: [extract-release-info, create-tarball] # update-release-hashes + if: always() && github.repository == 'redis/redis' + runs-on: ubuntu-latest + env: + TAG_NAME: ${{ needs.extract-release-info.outputs.tag_name }} + RELEASE_TYPE: ${{ needs.extract-release-info.outputs.release_type }} + SHA256: ${{ needs.create-tarball.outputs.sha256 }} + SIZE_MB: ${{ needs.create-tarball.outputs.size_mb }} + SIZE_WARNING: ${{ needs.create-tarball.outputs.size_warning }} + steps: + - name: Summary + run: | + { + echo "## Post-Release Automation Summary" + echo "" + echo "- **Release Tag:** ${TAG_NAME}" + echo "- **Release Type:** ${RELEASE_TYPE}" + echo "- **Tarball SHA256:** ${SHA256}" + echo "- **Tarball Size:** ${SIZE_MB} MB" + if [ "${SIZE_WARNING}" == "true" ]; then + echo "" + echo "> [!WARNING]" + echo "> Tarball size is outside expected range, check the logs for details." + fi + } >> $GITHUB_STEP_SUMMARY + + # - name: Send Slack notification + # run: | + # echo "TODO: Implement Slack notification" + # # This will require: + # # - Slack webhook URL or bot token (stored in secrets) + # # - Determine appropriate channel (e.g., #releases, #redis-releases) + # # - Craft message with release information and workflow status diff --git a/.gitignore b/.gitignore index 507aad8e0..5ed94f1da 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,6 @@ deps/lua/src/luac deps/lua/src/liblua.a deps/hdr_histogram/libhdrhistogram.a deps/fpconv/libfpconv.a -deps/fast_float/libfast_float.a tests/tls/* .make-* .prerequisites diff --git a/deps/Makefile b/deps/Makefile index c1d13bd85..60e0e569e 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -59,7 +59,6 @@ distclean: -(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true -(cd hdr_histogram && $(MAKE) clean) > /dev/null || true -(cd fpconv && $(MAKE) clean) > /dev/null || true - -(cd fast_float && $(MAKE) clean) > /dev/null || true -(cd xxhash && $(MAKE) clean) > /dev/null || true -(rm -f .make-*) @@ -95,12 +94,6 @@ fpconv: .make-prerequisites .PHONY: fpconv -fast_float: .make-prerequisites - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd fast_float && $(MAKE) libfast_float CFLAGS="$(DEPS_CFLAGS)" LDFLAGS="$(DEPS_LDFLAGS)" - -.PHONY: fast_float - XXHASH_CFLAGS = -fPIC $(DEPS_CFLAGS) xxhash: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) diff --git a/deps/fast_float/Makefile b/deps/fast_float/Makefile deleted file mode 100644 index e3acaa500..000000000 --- a/deps/fast_float/Makefile +++ /dev/null @@ -1,27 +0,0 @@ -# Fallback to gcc/g++ when $CC or $CXX is not in $PATH. -CC ?= gcc -CXX ?= g++ - -WARN=-Wall -OPT=-O3 -STD=-std=c++11 -DEFS=-DFASTFLOAT_ALLOWS_LEADING_PLUS - -FASTFLOAT_CFLAGS=$(WARN) $(OPT) $(STD) $(DEFS) $(CFLAGS) -FASTFLOAT_LDFLAGS=$(LDFLAGS) - -libfast_float: fast_float_strtod.o - $(AR) -r libfast_float.a fast_float_strtod.o - -32bit: FASTFLOAT_CFLAGS += -m32 -32bit: FASTFLOAT_LDFLAGS += -m32 -32bit: libfast_float - -fast_float_strtod.o: fast_float_strtod.cpp - $(CXX) $(FASTFLOAT_CFLAGS) -c fast_float_strtod.cpp $(FASTFLOAT_LDFLAGS) - -clean: - rm -f *.o - rm -f *.a - rm -f *.h.gch - rm -rf *.dSYM diff --git a/deps/fast_float/README.md b/deps/fast_float/README.md deleted file mode 100644 index 90462d3bf..000000000 --- a/deps/fast_float/README.md +++ /dev/null @@ -1,21 +0,0 @@ -README for fast_float v6.1.4 - ----------------------------------------------- - -We're using the fast_float library[1] in our (compiled-in) -floating-point fast_float_strtod implementation for faster and more -portable parsing of 64 decimal strings. - -The single file fast_float.h is an amalgamation of the entire library, -which can be (re)generated with the amalgamate.py script (from the -fast_float repository) via the command - -``` -git clone https://github.com/fastfloat/fast_float -cd fast_float -git checkout v6.1.4 -python3 ./script/amalgamate.py --license=MIT \ - > $REDIS_SRC/deps/fast_float/fast_float.h -``` - -[1]: https://github.com/fastfloat/fast_float diff --git a/deps/fast_float/fast_float.h b/deps/fast_float/fast_float.h deleted file mode 100644 index 81d9da50f..000000000 --- a/deps/fast_float/fast_float.h +++ /dev/null @@ -1,3838 +0,0 @@ -// fast_float by Daniel Lemire -// fast_float by João Paulo Magalhaes -// -// -// with contributions from Eugene Golushkov -// with contributions from Maksim Kita -// with contributions from Marcin Wojdyr -// with contributions from Neal Richardson -// with contributions from Tim Paine -// with contributions from Fabio Pellacini -// with contributions from Lénárd Szolnoki -// with contributions from Jan Pharago -// with contributions from Maya Warrier -// with contributions from Taha Khokhar -// -// -// MIT License Notice -// -// MIT License -// -// Copyright (c) 2021 The fast_float authors -// -// Permission is hereby granted, free of charge, to any -// person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the -// Software without restriction, including without -// limitation the rights to use, copy, modify, merge, -// publish, distribute, sublicense, and/or sell copies of -// the Software, and to permit persons to whom the Software -// is furnished to do so, subject to the following -// conditions: -// -// The above copyright notice and this permission notice -// shall be included in all copies or substantial portions -// of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -// - -#ifndef FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H -#define FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H - -#ifdef __has_include -#if __has_include() -#include -#endif -#endif - -// Testing for https://wg21.link/N3652, adopted in C++14 -#if __cpp_constexpr >= 201304 -#define FASTFLOAT_CONSTEXPR14 constexpr -#else -#define FASTFLOAT_CONSTEXPR14 -#endif - -#if defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L -#define FASTFLOAT_HAS_BIT_CAST 1 -#else -#define FASTFLOAT_HAS_BIT_CAST 0 -#endif - -#if defined(__cpp_lib_is_constant_evaluated) && \ - __cpp_lib_is_constant_evaluated >= 201811L -#define FASTFLOAT_HAS_IS_CONSTANT_EVALUATED 1 -#else -#define FASTFLOAT_HAS_IS_CONSTANT_EVALUATED 0 -#endif - -// Testing for relevant C++20 constexpr library features -#if FASTFLOAT_HAS_IS_CONSTANT_EVALUATED && FASTFLOAT_HAS_BIT_CAST && \ - __cpp_lib_constexpr_algorithms >= 201806L /*For std::copy and std::fill*/ -#define FASTFLOAT_CONSTEXPR20 constexpr -#define FASTFLOAT_IS_CONSTEXPR 1 -#else -#define FASTFLOAT_CONSTEXPR20 -#define FASTFLOAT_IS_CONSTEXPR 0 -#endif - -#endif // FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H - -#ifndef FASTFLOAT_FLOAT_COMMON_H -#define FASTFLOAT_FLOAT_COMMON_H - -#include -#include -#include -#include -#include -#include -#ifdef __has_include -#if __has_include() && (__cplusplus > 202002L || _MSVC_LANG > 202002L) -#include -#endif -#endif - -namespace fast_float { - -#define FASTFLOAT_JSONFMT (1 << 5) -#define FASTFLOAT_FORTRANFMT (1 << 6) - -enum chars_format { - scientific = 1 << 0, - fixed = 1 << 2, - hex = 1 << 3, - no_infnan = 1 << 4, - // RFC 8259: https://datatracker.ietf.org/doc/html/rfc8259#section-6 - json = FASTFLOAT_JSONFMT | fixed | scientific | no_infnan, - // Extension of RFC 8259 where, e.g., "inf" and "nan" are allowed. - json_or_infnan = FASTFLOAT_JSONFMT | fixed | scientific, - fortran = FASTFLOAT_FORTRANFMT | fixed | scientific, - general = fixed | scientific -}; - -template struct from_chars_result_t { - UC const *ptr; - std::errc ec; -}; -using from_chars_result = from_chars_result_t; - -template struct parse_options_t { - constexpr explicit parse_options_t(chars_format fmt = chars_format::general, - UC dot = UC('.')) - : format(fmt), decimal_point(dot) {} - - /** Which number formats are accepted */ - chars_format format; - /** The character used as decimal point */ - UC decimal_point; -}; -using parse_options = parse_options_t; - -} // namespace fast_float - -#if FASTFLOAT_HAS_BIT_CAST -#include -#endif - -#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(__amd64) || defined(__aarch64__) || defined(_M_ARM64) || \ - defined(__MINGW64__) || defined(__s390x__) || \ - (defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \ - defined(__PPC64LE__)) || \ - defined(__loongarch64)) -#define FASTFLOAT_64BIT 1 -#elif (defined(__i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__arm__) || defined(_M_ARM) || defined(__ppc__) || \ - defined(__MINGW32__) || defined(__EMSCRIPTEN__)) -#define FASTFLOAT_32BIT 1 -#else - // Need to check incrementally, since SIZE_MAX is a size_t, avoid overflow. -// We can never tell the register width, but the SIZE_MAX is a good -// approximation. UINTPTR_MAX and INTPTR_MAX are optional, so avoid them for max -// portability. -#if SIZE_MAX == 0xffff -#error Unknown platform (16-bit, unsupported) -#elif SIZE_MAX == 0xffffffff -#define FASTFLOAT_32BIT 1 -#elif SIZE_MAX == 0xffffffffffffffff -#define FASTFLOAT_64BIT 1 -#else -#error Unknown platform (not 32-bit, not 64-bit?) -#endif -#endif - -#if ((defined(_WIN32) || defined(_WIN64)) && !defined(__clang__)) || \ - (defined(_M_ARM64) && !defined(__MINGW32__)) -#include -#endif - -#if defined(_MSC_VER) && !defined(__clang__) -#define FASTFLOAT_VISUAL_STUDIO 1 -#endif - -#if defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__ -#define FASTFLOAT_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) -#elif defined _WIN32 -#define FASTFLOAT_IS_BIG_ENDIAN 0 -#else -#if defined(__APPLE__) || defined(__FreeBSD__) -#include -#elif defined(sun) || defined(__sun) -#include -#elif defined(__MVS__) -#include -#else -#ifdef __has_include -#if __has_include() -#include -#endif //__has_include() -#endif //__has_include -#endif -# -#ifndef __BYTE_ORDER__ -// safe choice -#define FASTFLOAT_IS_BIG_ENDIAN 0 -#endif -# -#ifndef __ORDER_LITTLE_ENDIAN__ -// safe choice -#define FASTFLOAT_IS_BIG_ENDIAN 0 -#endif -# -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define FASTFLOAT_IS_BIG_ENDIAN 0 -#else -#define FASTFLOAT_IS_BIG_ENDIAN 1 -#endif -#endif - -#if defined(__SSE2__) || (defined(FASTFLOAT_VISUAL_STUDIO) && \ - (defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && _M_IX86_FP == 2))) -#define FASTFLOAT_SSE2 1 -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#define FASTFLOAT_NEON 1 -#endif - -#if defined(FASTFLOAT_SSE2) || defined(FASTFLOAT_NEON) -#define FASTFLOAT_HAS_SIMD 1 -#endif - -#if defined(__GNUC__) -// disable -Wcast-align=strict (GCC only) -#define FASTFLOAT_SIMD_DISABLE_WARNINGS \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wcast-align\"") -#else -#define FASTFLOAT_SIMD_DISABLE_WARNINGS -#endif - -#if defined(__GNUC__) -#define FASTFLOAT_SIMD_RESTORE_WARNINGS _Pragma("GCC diagnostic pop") -#else -#define FASTFLOAT_SIMD_RESTORE_WARNINGS -#endif - -#ifdef FASTFLOAT_VISUAL_STUDIO -#define fastfloat_really_inline __forceinline -#else -#define fastfloat_really_inline inline __attribute__((always_inline)) -#endif - -#ifndef FASTFLOAT_ASSERT -#define FASTFLOAT_ASSERT(x) \ - { ((void)(x)); } -#endif - -#ifndef FASTFLOAT_DEBUG_ASSERT -#define FASTFLOAT_DEBUG_ASSERT(x) \ - { ((void)(x)); } -#endif - -// rust style `try!()` macro, or `?` operator -#define FASTFLOAT_TRY(x) \ - { \ - if (!(x)) \ - return false; \ - } - -#define FASTFLOAT_ENABLE_IF(...) \ - typename std::enable_if<(__VA_ARGS__), int>::type - -namespace fast_float { - -fastfloat_really_inline constexpr bool cpp20_and_in_constexpr() { -#if FASTFLOAT_HAS_IS_CONSTANT_EVALUATED - return std::is_constant_evaluated(); -#else - return false; -#endif -} - -template -fastfloat_really_inline constexpr bool is_supported_float_type() { - return std::is_same::value || std::is_same::value -#if __STDCPP_FLOAT32_T__ - || std::is_same::value -#endif -#if __STDCPP_FLOAT64_T__ - || std::is_same::value -#endif - ; -} - -template -fastfloat_really_inline constexpr bool is_supported_char_type() { - return std::is_same::value || std::is_same::value || - std::is_same::value || std::is_same::value; -} - -// Compares two ASCII strings in a case insensitive manner. -template -inline FASTFLOAT_CONSTEXPR14 bool -fastfloat_strncasecmp(UC const *input1, UC const *input2, size_t length) { - char running_diff{0}; - for (size_t i = 0; i < length; ++i) { - running_diff |= (char(input1[i]) ^ char(input2[i])); - } - return (running_diff == 0) || (running_diff == 32); -} - -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif - -// a pointer and a length to a contiguous block of memory -template struct span { - const T *ptr; - size_t length; - constexpr span(const T *_ptr, size_t _length) : ptr(_ptr), length(_length) {} - constexpr span() : ptr(nullptr), length(0) {} - - constexpr size_t len() const noexcept { return length; } - - FASTFLOAT_CONSTEXPR14 const T &operator[](size_t index) const noexcept { - FASTFLOAT_DEBUG_ASSERT(index < length); - return ptr[index]; - } -}; - -struct value128 { - uint64_t low; - uint64_t high; - constexpr value128(uint64_t _low, uint64_t _high) : low(_low), high(_high) {} - constexpr value128() : low(0), high(0) {} -}; - -/* Helper C++14 constexpr generic implementation of leading_zeroes */ -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 int -leading_zeroes_generic(uint64_t input_num, int last_bit = 0) { - if (input_num & uint64_t(0xffffffff00000000)) { - input_num >>= 32; - last_bit |= 32; - } - if (input_num & uint64_t(0xffff0000)) { - input_num >>= 16; - last_bit |= 16; - } - if (input_num & uint64_t(0xff00)) { - input_num >>= 8; - last_bit |= 8; - } - if (input_num & uint64_t(0xf0)) { - input_num >>= 4; - last_bit |= 4; - } - if (input_num & uint64_t(0xc)) { - input_num >>= 2; - last_bit |= 2; - } - if (input_num & uint64_t(0x2)) { /* input_num >>= 1; */ - last_bit |= 1; - } - return 63 - last_bit; -} - -/* result might be undefined when input_num is zero */ -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 int -leading_zeroes(uint64_t input_num) { - assert(input_num > 0); - if (cpp20_and_in_constexpr()) { - return leading_zeroes_generic(input_num); - } -#ifdef FASTFLOAT_VISUAL_STUDIO -#if defined(_M_X64) || defined(_M_ARM64) - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - _BitScanReverse64(&leading_zero, input_num); - return (int)(63 - leading_zero); -#else - return leading_zeroes_generic(input_num); -#endif -#else - return __builtin_clzll(input_num); -#endif -} - -// slow emulation routine for 32-bit -fastfloat_really_inline constexpr uint64_t emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} - -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint64_t -umul128_generic(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = (uint64_t)(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + (uint64_t)(lo < bd); - return lo; -} - -#ifdef FASTFLOAT_32BIT - -// slow emulation routine for 32-bit -#if !defined(__MINGW64__) -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint64_t _umul128(uint64_t ab, - uint64_t cd, - uint64_t *hi) { - return umul128_generic(ab, cd, hi); -} -#endif // !__MINGW64__ - -#endif // FASTFLOAT_32BIT - -// compute 64-bit a*b -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 value128 -full_multiplication(uint64_t a, uint64_t b) { - if (cpp20_and_in_constexpr()) { - value128 answer; - answer.low = umul128_generic(a, b, &answer.high); - return answer; - } - value128 answer; -#if defined(_M_ARM64) && !defined(__MINGW32__) - // ARM64 has native support for 64-bit multiplications, no need to emulate - // But MinGW on ARM64 doesn't have native support for 64-bit multiplications - answer.high = __umulh(a, b); - answer.low = a * b; -#elif defined(FASTFLOAT_32BIT) || (defined(_WIN64) && !defined(__clang__)) - answer.low = _umul128(a, b, &answer.high); // _umul128 not available on ARM64 -#elif defined(FASTFLOAT_64BIT) && defined(__SIZEOF_INT128__) - __uint128_t r = ((__uint128_t)a) * b; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#else - answer.low = umul128_generic(a, b, &answer.high); -#endif - return answer; -} - -struct adjusted_mantissa { - uint64_t mantissa{0}; - int32_t power2{0}; // a negative value indicates an invalid result - adjusted_mantissa() = default; - constexpr bool operator==(const adjusted_mantissa &o) const { - return mantissa == o.mantissa && power2 == o.power2; - } - constexpr bool operator!=(const adjusted_mantissa &o) const { - return mantissa != o.mantissa || power2 != o.power2; - } -}; - -// Bias so we can get the real exponent with an invalid adjusted_mantissa. -constexpr static int32_t invalid_am_bias = -0x8000; - -// used for binary_format_lookup_tables::max_mantissa -constexpr uint64_t constant_55555 = 5 * 5 * 5 * 5 * 5; - -template struct binary_format_lookup_tables; - -template struct binary_format : binary_format_lookup_tables { - using equiv_uint = - typename std::conditional::type; - - static inline constexpr int mantissa_explicit_bits(); - static inline constexpr int minimum_exponent(); - static inline constexpr int infinite_power(); - static inline constexpr int sign_index(); - static inline constexpr int - min_exponent_fast_path(); // used when fegetround() == FE_TONEAREST - static inline constexpr int max_exponent_fast_path(); - static inline constexpr int max_exponent_round_to_even(); - static inline constexpr int min_exponent_round_to_even(); - static inline constexpr uint64_t max_mantissa_fast_path(int64_t power); - static inline constexpr uint64_t - max_mantissa_fast_path(); // used when fegetround() == FE_TONEAREST - static inline constexpr int largest_power_of_ten(); - static inline constexpr int smallest_power_of_ten(); - static inline constexpr T exact_power_of_ten(int64_t power); - static inline constexpr size_t max_digits(); - static inline constexpr equiv_uint exponent_mask(); - static inline constexpr equiv_uint mantissa_mask(); - static inline constexpr equiv_uint hidden_bit_mask(); -}; - -template struct binary_format_lookup_tables { - static constexpr double powers_of_ten[] = { - 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, - 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22}; - - // Largest integer value v so that (5**index * v) <= 1<<53. - // 0x20000000000000 == 1 << 53 - static constexpr uint64_t max_mantissa[] = { - 0x20000000000000, - 0x20000000000000 / 5, - 0x20000000000000 / (5 * 5), - 0x20000000000000 / (5 * 5 * 5), - 0x20000000000000 / (5 * 5 * 5 * 5), - 0x20000000000000 / (constant_55555), - 0x20000000000000 / (constant_55555 * 5), - 0x20000000000000 / (constant_55555 * 5 * 5), - 0x20000000000000 / (constant_55555 * 5 * 5 * 5), - 0x20000000000000 / (constant_55555 * 5 * 5 * 5 * 5), - 0x20000000000000 / (constant_55555 * constant_55555), - 0x20000000000000 / (constant_55555 * constant_55555 * 5), - 0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5), - 0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5 * 5), - 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555), - 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * 5), - 0x20000000000000 / - (constant_55555 * constant_55555 * constant_55555 * 5 * 5), - 0x20000000000000 / - (constant_55555 * constant_55555 * constant_55555 * 5 * 5 * 5), - 0x20000000000000 / - (constant_55555 * constant_55555 * constant_55555 * 5 * 5 * 5 * 5), - 0x20000000000000 / - (constant_55555 * constant_55555 * constant_55555 * constant_55555), - 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * - constant_55555 * 5), - 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * - constant_55555 * 5 * 5), - 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * - constant_55555 * 5 * 5 * 5), - 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * - constant_55555 * 5 * 5 * 5 * 5)}; -}; - -template -constexpr double binary_format_lookup_tables::powers_of_ten[]; - -template -constexpr uint64_t binary_format_lookup_tables::max_mantissa[]; - -template struct binary_format_lookup_tables { - static constexpr float powers_of_ten[] = {1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - - // Largest integer value v so that (5**index * v) <= 1<<24. - // 0x1000000 == 1<<24 - static constexpr uint64_t max_mantissa[] = { - 0x1000000, - 0x1000000 / 5, - 0x1000000 / (5 * 5), - 0x1000000 / (5 * 5 * 5), - 0x1000000 / (5 * 5 * 5 * 5), - 0x1000000 / (constant_55555), - 0x1000000 / (constant_55555 * 5), - 0x1000000 / (constant_55555 * 5 * 5), - 0x1000000 / (constant_55555 * 5 * 5 * 5), - 0x1000000 / (constant_55555 * 5 * 5 * 5 * 5), - 0x1000000 / (constant_55555 * constant_55555), - 0x1000000 / (constant_55555 * constant_55555 * 5)}; -}; - -template -constexpr float binary_format_lookup_tables::powers_of_ten[]; - -template -constexpr uint64_t binary_format_lookup_tables::max_mantissa[]; - -template <> -inline constexpr int binary_format::min_exponent_fast_path() { -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - return 0; -#else - return -22; -#endif -} - -template <> -inline constexpr int binary_format::min_exponent_fast_path() { -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - return 0; -#else - return -10; -#endif -} - -template <> -inline constexpr int binary_format::mantissa_explicit_bits() { - return 52; -} -template <> -inline constexpr int binary_format::mantissa_explicit_bits() { - return 23; -} - -template <> -inline constexpr int binary_format::max_exponent_round_to_even() { - return 23; -} - -template <> -inline constexpr int binary_format::max_exponent_round_to_even() { - return 10; -} - -template <> -inline constexpr int binary_format::min_exponent_round_to_even() { - return -4; -} - -template <> -inline constexpr int binary_format::min_exponent_round_to_even() { - return -17; -} - -template <> inline constexpr int binary_format::minimum_exponent() { - return -1023; -} -template <> inline constexpr int binary_format::minimum_exponent() { - return -127; -} - -template <> inline constexpr int binary_format::infinite_power() { - return 0x7FF; -} -template <> inline constexpr int binary_format::infinite_power() { - return 0xFF; -} - -template <> inline constexpr int binary_format::sign_index() { - return 63; -} -template <> inline constexpr int binary_format::sign_index() { - return 31; -} - -template <> -inline constexpr int binary_format::max_exponent_fast_path() { - return 22; -} -template <> -inline constexpr int binary_format::max_exponent_fast_path() { - return 10; -} - -template <> -inline constexpr uint64_t binary_format::max_mantissa_fast_path() { - return uint64_t(2) << mantissa_explicit_bits(); -} -template <> -inline constexpr uint64_t -binary_format::max_mantissa_fast_path(int64_t power) { - // caller is responsible to ensure that - // power >= 0 && power <= 22 - // - // Work around clang bug https://godbolt.org/z/zedh7rrhc - return (void)max_mantissa[0], max_mantissa[power]; -} -template <> -inline constexpr uint64_t binary_format::max_mantissa_fast_path() { - return uint64_t(2) << mantissa_explicit_bits(); -} -template <> -inline constexpr uint64_t -binary_format::max_mantissa_fast_path(int64_t power) { - // caller is responsible to ensure that - // power >= 0 && power <= 10 - // - // Work around clang bug https://godbolt.org/z/zedh7rrhc - return (void)max_mantissa[0], max_mantissa[power]; -} - -template <> -inline constexpr double -binary_format::exact_power_of_ten(int64_t power) { - // Work around clang bug https://godbolt.org/z/zedh7rrhc - return (void)powers_of_ten[0], powers_of_ten[power]; -} -template <> -inline constexpr float binary_format::exact_power_of_ten(int64_t power) { - // Work around clang bug https://godbolt.org/z/zedh7rrhc - return (void)powers_of_ten[0], powers_of_ten[power]; -} - -template <> inline constexpr int binary_format::largest_power_of_ten() { - return 308; -} -template <> inline constexpr int binary_format::largest_power_of_ten() { - return 38; -} - -template <> -inline constexpr int binary_format::smallest_power_of_ten() { - return -342; -} -template <> inline constexpr int binary_format::smallest_power_of_ten() { - return -64; -} - -template <> inline constexpr size_t binary_format::max_digits() { - return 769; -} -template <> inline constexpr size_t binary_format::max_digits() { - return 114; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::exponent_mask() { - return 0x7F800000; -} -template <> -inline constexpr binary_format::equiv_uint -binary_format::exponent_mask() { - return 0x7FF0000000000000; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::mantissa_mask() { - return 0x007FFFFF; -} -template <> -inline constexpr binary_format::equiv_uint -binary_format::mantissa_mask() { - return 0x000FFFFFFFFFFFFF; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::hidden_bit_mask() { - return 0x00800000; -} -template <> -inline constexpr binary_format::equiv_uint -binary_format::hidden_bit_mask() { - return 0x0010000000000000; -} - -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void -to_float(bool negative, adjusted_mantissa am, T &value) { - using fastfloat_uint = typename binary_format::equiv_uint; - fastfloat_uint word = (fastfloat_uint)am.mantissa; - word |= fastfloat_uint(am.power2) - << binary_format::mantissa_explicit_bits(); - word |= fastfloat_uint(negative) << binary_format::sign_index(); -#if FASTFLOAT_HAS_BIT_CAST - value = std::bit_cast(word); -#else - ::memcpy(&value, &word, sizeof(T)); -#endif -} - -#ifdef FASTFLOAT_SKIP_WHITE_SPACE // disabled by default -template struct space_lut { - static constexpr bool value[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -}; - -template constexpr bool space_lut::value[]; - -inline constexpr bool is_space(uint8_t c) { return space_lut<>::value[c]; } -#endif - -template static constexpr uint64_t int_cmp_zeros() { - static_assert((sizeof(UC) == 1) || (sizeof(UC) == 2) || (sizeof(UC) == 4), - "Unsupported character size"); - return (sizeof(UC) == 1) ? 0x3030303030303030 - : (sizeof(UC) == 2) - ? (uint64_t(UC('0')) << 48 | uint64_t(UC('0')) << 32 | - uint64_t(UC('0')) << 16 | UC('0')) - : (uint64_t(UC('0')) << 32 | UC('0')); -} -template static constexpr int int_cmp_len() { - return sizeof(uint64_t) / sizeof(UC); -} -template static constexpr UC const *str_const_nan() { - return nullptr; -} -template <> constexpr char const *str_const_nan() { return "nan"; } -template <> constexpr wchar_t const *str_const_nan() { return L"nan"; } -template <> constexpr char16_t const *str_const_nan() { - return u"nan"; -} -template <> constexpr char32_t const *str_const_nan() { - return U"nan"; -} -template static constexpr UC const *str_const_inf() { - return nullptr; -} -template <> constexpr char const *str_const_inf() { return "infinity"; } -template <> constexpr wchar_t const *str_const_inf() { - return L"infinity"; -} -template <> constexpr char16_t const *str_const_inf() { - return u"infinity"; -} -template <> constexpr char32_t const *str_const_inf() { - return U"infinity"; -} - -template struct int_luts { - static constexpr uint8_t chdigit[] = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, - 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, - 35, 255, 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, 15, 16, 17, - 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, - 33, 34, 35, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255}; - - static constexpr size_t maxdigits_u64[] = { - 64, 41, 32, 28, 25, 23, 22, 21, 20, 19, 18, 18, 17, 17, 16, 16, 16, 16, - 15, 15, 15, 15, 14, 14, 14, 14, 14, 14, 14, 13, 13, 13, 13, 13, 13}; - - static constexpr uint64_t min_safe_u64[] = { - 9223372036854775808ull, 12157665459056928801ull, 4611686018427387904, - 7450580596923828125, 4738381338321616896, 3909821048582988049, - 9223372036854775808ull, 12157665459056928801ull, 10000000000000000000ull, - 5559917313492231481, 2218611106740436992, 8650415919381337933, - 2177953337809371136, 6568408355712890625, 1152921504606846976, - 2862423051509815793, 6746640616477458432, 15181127029874798299ull, - 1638400000000000000, 3243919932521508681, 6221821273427820544, - 11592836324538749809ull, 876488338465357824, 1490116119384765625, - 2481152873203736576, 4052555153018976267, 6502111422497947648, - 10260628712958602189ull, 15943230000000000000ull, 787662783788549761, - 1152921504606846976, 1667889514952984961, 2386420683693101056, - 3379220508056640625, 4738381338321616896}; -}; - -template constexpr uint8_t int_luts::chdigit[]; - -template constexpr size_t int_luts::maxdigits_u64[]; - -template constexpr uint64_t int_luts::min_safe_u64[]; - -template -fastfloat_really_inline constexpr uint8_t ch_to_digit(UC c) { - return int_luts<>::chdigit[static_cast(c)]; -} - -fastfloat_really_inline constexpr size_t max_digits_u64(int base) { - return int_luts<>::maxdigits_u64[base - 2]; -} - -// If a u64 is exactly max_digits_u64() in length, this is -// the value below which it has definitely overflowed. -fastfloat_really_inline constexpr uint64_t min_safe_u64(int base) { - return int_luts<>::min_safe_u64[base - 2]; -} - -} // namespace fast_float - -#endif - - -#ifndef FASTFLOAT_FAST_FLOAT_H -#define FASTFLOAT_FAST_FLOAT_H - - -namespace fast_float { -/** - * This function parses the character sequence [first,last) for a number. It - * parses floating-point numbers expecting a locale-indepent format equivalent - * to what is used by std::strtod in the default ("C") locale. The resulting - * floating-point value is the closest floating-point values (using either float - * or double), using the "round to even" convention for values that would - * otherwise fall right in-between two values. That is, we provide exact parsing - * according to the IEEE standard. - * - * Given a successful parse, the pointer (`ptr`) in the returned value is set to - * point right after the parsed number, and the `value` referenced is set to the - * parsed value. In case of error, the returned `ec` contains a representative - * error, otherwise the default (`std::errc()`) value is stored. - * - * The implementation does not throw and does not allocate memory (e.g., with - * `new` or `malloc`). - * - * Like the C++17 standard, the `fast_float::from_chars` functions take an - * optional last argument of the type `fast_float::chars_format`. It is a bitset - * value: we check whether `fmt & fast_float::chars_format::fixed` and `fmt & - * fast_float::chars_format::scientific` are set to determine whether we allow - * the fixed point and scientific notation respectively. The default is - * `fast_float::chars_format::general` which allows both `fixed` and - * `scientific`. - */ -template ())> -FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars(UC const *first, UC const *last, T &value, - chars_format fmt = chars_format::general) noexcept; - -/** - * Like from_chars, but accepts an `options` argument to govern number parsing. - */ -template -FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars_advanced(UC const *first, UC const *last, T &value, - parse_options_t options) noexcept; -/** - * from_chars for integer types. - */ -template ())> -FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars(UC const *first, UC const *last, T &value, int base = 10) noexcept; - -} // namespace fast_float -#endif // FASTFLOAT_FAST_FLOAT_H - -#ifndef FASTFLOAT_ASCII_NUMBER_H -#define FASTFLOAT_ASCII_NUMBER_H - -#include -#include -#include -#include -#include -#include - - -#ifdef FASTFLOAT_SSE2 -#include -#endif - -#ifdef FASTFLOAT_NEON -#include -#endif - -namespace fast_float { - -template fastfloat_really_inline constexpr bool has_simd_opt() { -#ifdef FASTFLOAT_HAS_SIMD - return std::is_same::value; -#else - return false; -#endif -} - -// Next function can be micro-optimized, but compilers are entirely -// able to optimize it well. -template -fastfloat_really_inline constexpr bool is_integer(UC c) noexcept { - return !(c > UC('9') || c < UC('0')); -} - -fastfloat_really_inline constexpr uint64_t byteswap(uint64_t val) { - return (val & 0xFF00000000000000) >> 56 | (val & 0x00FF000000000000) >> 40 | - (val & 0x0000FF0000000000) >> 24 | (val & 0x000000FF00000000) >> 8 | - (val & 0x00000000FF000000) << 8 | (val & 0x0000000000FF0000) << 24 | - (val & 0x000000000000FF00) << 40 | (val & 0x00000000000000FF) << 56; -} - -// Read 8 UC into a u64. Truncates UC if not char. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t -read8_to_u64(const UC *chars) { - if (cpp20_and_in_constexpr() || !std::is_same::value) { - uint64_t val = 0; - for (int i = 0; i < 8; ++i) { - val |= uint64_t(uint8_t(*chars)) << (i * 8); - ++chars; - } - return val; - } - uint64_t val; - ::memcpy(&val, chars, sizeof(uint64_t)); -#if FASTFLOAT_IS_BIG_ENDIAN == 1 - // Need to read as-if the number was in little-endian order. - val = byteswap(val); -#endif - return val; -} - -#ifdef FASTFLOAT_SSE2 - -fastfloat_really_inline uint64_t simd_read8_to_u64(const __m128i data) { - FASTFLOAT_SIMD_DISABLE_WARNINGS - const __m128i packed = _mm_packus_epi16(data, data); -#ifdef FASTFLOAT_64BIT - return uint64_t(_mm_cvtsi128_si64(packed)); -#else - uint64_t value; - // Visual Studio + older versions of GCC don't support _mm_storeu_si64 - _mm_storel_epi64(reinterpret_cast<__m128i *>(&value), packed); - return value; -#endif - FASTFLOAT_SIMD_RESTORE_WARNINGS -} - -fastfloat_really_inline uint64_t simd_read8_to_u64(const char16_t *chars) { - FASTFLOAT_SIMD_DISABLE_WARNINGS - return simd_read8_to_u64( - _mm_loadu_si128(reinterpret_cast(chars))); - FASTFLOAT_SIMD_RESTORE_WARNINGS -} - -#elif defined(FASTFLOAT_NEON) - -fastfloat_really_inline uint64_t simd_read8_to_u64(const uint16x8_t data) { - FASTFLOAT_SIMD_DISABLE_WARNINGS - uint8x8_t utf8_packed = vmovn_u16(data); - return vget_lane_u64(vreinterpret_u64_u8(utf8_packed), 0); - FASTFLOAT_SIMD_RESTORE_WARNINGS -} - -fastfloat_really_inline uint64_t simd_read8_to_u64(const char16_t *chars) { - FASTFLOAT_SIMD_DISABLE_WARNINGS - return simd_read8_to_u64( - vld1q_u16(reinterpret_cast(chars))); - FASTFLOAT_SIMD_RESTORE_WARNINGS -} - -#endif // FASTFLOAT_SSE2 - -// MSVC SFINAE is broken pre-VS2017 -#if defined(_MSC_VER) && _MSC_VER <= 1900 -template -#else -template ()) = 0> -#endif -// dummy for compile -uint64_t simd_read8_to_u64(UC const *) { - return 0; -} - -// credit @aqrit -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint32_t -parse_eight_digits_unrolled(uint64_t val) { - const uint64_t mask = 0x000000FF000000FF; - const uint64_t mul1 = 0x000F424000000064; // 100 + (1000000ULL << 32) - const uint64_t mul2 = 0x0000271000000001; // 1 + (10000ULL << 32) - val -= 0x3030303030303030; - val = (val * 10) + (val >> 8); // val = (val * 2561) >> 8; - val = (((val & mask) * mul1) + (((val >> 16) & mask) * mul2)) >> 32; - return uint32_t(val); -} - -// Call this if chars are definitely 8 digits. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint32_t -parse_eight_digits_unrolled(UC const *chars) noexcept { - if (cpp20_and_in_constexpr() || !has_simd_opt()) { - return parse_eight_digits_unrolled(read8_to_u64(chars)); // truncation okay - } - return parse_eight_digits_unrolled(simd_read8_to_u64(chars)); -} - -// credit @aqrit -fastfloat_really_inline constexpr bool -is_made_of_eight_digits_fast(uint64_t val) noexcept { - return !((((val + 0x4646464646464646) | (val - 0x3030303030303030)) & - 0x8080808080808080)); -} - -#ifdef FASTFLOAT_HAS_SIMD - -// Call this if chars might not be 8 digits. -// Using this style (instead of is_made_of_eight_digits_fast() then -// parse_eight_digits_unrolled()) ensures we don't load SIMD registers twice. -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool -simd_parse_if_eight_digits_unrolled(const char16_t *chars, - uint64_t &i) noexcept { - if (cpp20_and_in_constexpr()) { - return false; - } -#ifdef FASTFLOAT_SSE2 - FASTFLOAT_SIMD_DISABLE_WARNINGS - const __m128i data = - _mm_loadu_si128(reinterpret_cast(chars)); - - // (x - '0') <= 9 - // http://0x80.pl/articles/simd-parsing-int-sequences.html - const __m128i t0 = _mm_add_epi16(data, _mm_set1_epi16(32720)); - const __m128i t1 = _mm_cmpgt_epi16(t0, _mm_set1_epi16(-32759)); - - if (_mm_movemask_epi8(t1) == 0) { - i = i * 100000000 + parse_eight_digits_unrolled(simd_read8_to_u64(data)); - return true; - } else - return false; - FASTFLOAT_SIMD_RESTORE_WARNINGS -#elif defined(FASTFLOAT_NEON) - FASTFLOAT_SIMD_DISABLE_WARNINGS - const uint16x8_t data = vld1q_u16(reinterpret_cast(chars)); - - // (x - '0') <= 9 - // http://0x80.pl/articles/simd-parsing-int-sequences.html - const uint16x8_t t0 = vsubq_u16(data, vmovq_n_u16('0')); - const uint16x8_t mask = vcltq_u16(t0, vmovq_n_u16('9' - '0' + 1)); - - if (vminvq_u16(mask) == 0xFFFF) { - i = i * 100000000 + parse_eight_digits_unrolled(simd_read8_to_u64(data)); - return true; - } else - return false; - FASTFLOAT_SIMD_RESTORE_WARNINGS -#else - (void)chars; - (void)i; - return false; -#endif // FASTFLOAT_SSE2 -} - -#endif // FASTFLOAT_HAS_SIMD - -// MSVC SFINAE is broken pre-VS2017 -#if defined(_MSC_VER) && _MSC_VER <= 1900 -template -#else -template ()) = 0> -#endif -// dummy for compile -bool simd_parse_if_eight_digits_unrolled(UC const *, uint64_t &) { - return 0; -} - -template ::value) = 0> -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void -loop_parse_if_eight_digits(const UC *&p, const UC *const pend, uint64_t &i) { - if (!has_simd_opt()) { - return; - } - while ((std::distance(p, pend) >= 8) && - simd_parse_if_eight_digits_unrolled( - p, i)) { // in rare cases, this will overflow, but that's ok - p += 8; - } -} - -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void -loop_parse_if_eight_digits(const char *&p, const char *const pend, - uint64_t &i) { - // optimizes better than parse_if_eight_digits_unrolled() for UC = char. - while ((std::distance(p, pend) >= 8) && - is_made_of_eight_digits_fast(read8_to_u64(p))) { - i = i * 100000000 + - parse_eight_digits_unrolled(read8_to_u64( - p)); // in rare cases, this will overflow, but that's ok - p += 8; - } -} - -enum class parse_error { - no_error, - // [JSON-only] The minus sign must be followed by an integer. - missing_integer_after_sign, - // A sign must be followed by an integer or dot. - missing_integer_or_dot_after_sign, - // [JSON-only] The integer part must not have leading zeros. - leading_zeros_in_integer_part, - // [JSON-only] The integer part must have at least one digit. - no_digits_in_integer_part, - // [JSON-only] If there is a decimal point, there must be digits in the - // fractional part. - no_digits_in_fractional_part, - // The mantissa must have at least one digit. - no_digits_in_mantissa, - // Scientific notation requires an exponential part. - missing_exponential_part, -}; - -template struct parsed_number_string_t { - int64_t exponent{0}; - uint64_t mantissa{0}; - UC const *lastmatch{nullptr}; - bool negative{false}; - bool valid{false}; - bool too_many_digits{false}; - // contains the range of the significant digits - span integer{}; // non-nullable - span fraction{}; // nullable - parse_error error{parse_error::no_error}; -}; - -using byte_span = span; -using parsed_number_string = parsed_number_string_t; - -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t -report_parse_error(UC const *p, parse_error error) { - parsed_number_string_t answer; - answer.valid = false; - answer.lastmatch = p; - answer.error = error; - return answer; -} - -// Assuming that you use no more than 19 digits, this will -// parse an ASCII string. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t -parse_number_string(UC const *p, UC const *pend, - parse_options_t options) noexcept { - chars_format const fmt = options.format; - UC const decimal_point = options.decimal_point; - - parsed_number_string_t answer; - answer.valid = false; - answer.too_many_digits = false; - answer.negative = (*p == UC('-')); -#ifdef FASTFLOAT_ALLOWS_LEADING_PLUS // disabled by default - if ((*p == UC('-')) || (!(fmt & FASTFLOAT_JSONFMT) && *p == UC('+'))) { -#else - if (*p == UC('-')) { // C++17 20.19.3.(7.1) explicitly forbids '+' sign here -#endif - ++p; - if (p == pend) { - return report_parse_error( - p, parse_error::missing_integer_or_dot_after_sign); - } - if (fmt & FASTFLOAT_JSONFMT) { - if (!is_integer(*p)) { // a sign must be followed by an integer - return report_parse_error(p, - parse_error::missing_integer_after_sign); - } - } else { - if (!is_integer(*p) && - (*p != - decimal_point)) { // a sign must be followed by an integer or the dot - return report_parse_error( - p, parse_error::missing_integer_or_dot_after_sign); - } - } - } - UC const *const start_digits = p; - - uint64_t i = 0; // an unsigned int avoids signed overflows (which are bad) - - while ((p != pend) && is_integer(*p)) { - // a multiplication by 10 is cheaper than an arbitrary integer - // multiplication - i = 10 * i + - uint64_t(*p - - UC('0')); // might overflow, we will handle the overflow later - ++p; - } - UC const *const end_of_integer_part = p; - int64_t digit_count = int64_t(end_of_integer_part - start_digits); - answer.integer = span(start_digits, size_t(digit_count)); - if (fmt & FASTFLOAT_JSONFMT) { - // at least 1 digit in integer part, without leading zeros - if (digit_count == 0) { - return report_parse_error(p, parse_error::no_digits_in_integer_part); - } - if ((start_digits[0] == UC('0') && digit_count > 1)) { - return report_parse_error(start_digits, - parse_error::leading_zeros_in_integer_part); - } - } - - int64_t exponent = 0; - const bool has_decimal_point = (p != pend) && (*p == decimal_point); - if (has_decimal_point) { - ++p; - UC const *before = p; - // can occur at most twice without overflowing, but let it occur more, since - // for integers with many digits, digit parsing is the primary bottleneck. - loop_parse_if_eight_digits(p, pend, i); - - while ((p != pend) && is_integer(*p)) { - uint8_t digit = uint8_t(*p - UC('0')); - ++p; - i = i * 10 + digit; // in rare cases, this will overflow, but that's ok - } - exponent = before - p; - answer.fraction = span(before, size_t(p - before)); - digit_count -= exponent; - } - if (fmt & FASTFLOAT_JSONFMT) { - // at least 1 digit in fractional part - if (has_decimal_point && exponent == 0) { - return report_parse_error(p, - parse_error::no_digits_in_fractional_part); - } - } else if (digit_count == - 0) { // we must have encountered at least one integer! - return report_parse_error(p, parse_error::no_digits_in_mantissa); - } - int64_t exp_number = 0; // explicit exponential part - if (((fmt & chars_format::scientific) && (p != pend) && - ((UC('e') == *p) || (UC('E') == *p))) || - ((fmt & FASTFLOAT_FORTRANFMT) && (p != pend) && - ((UC('+') == *p) || (UC('-') == *p) || (UC('d') == *p) || - (UC('D') == *p)))) { - UC const *location_of_e = p; - if ((UC('e') == *p) || (UC('E') == *p) || (UC('d') == *p) || - (UC('D') == *p)) { - ++p; - } - bool neg_exp = false; - if ((p != pend) && (UC('-') == *p)) { - neg_exp = true; - ++p; - } else if ((p != pend) && - (UC('+') == - *p)) { // '+' on exponent is allowed by C++17 20.19.3.(7.1) - ++p; - } - if ((p == pend) || !is_integer(*p)) { - if (!(fmt & chars_format::fixed)) { - // The exponential part is invalid for scientific notation, so it must - // be a trailing token for fixed notation. However, fixed notation is - // disabled, so report a scientific notation error. - return report_parse_error(p, parse_error::missing_exponential_part); - } - // Otherwise, we will be ignoring the 'e'. - p = location_of_e; - } else { - while ((p != pend) && is_integer(*p)) { - uint8_t digit = uint8_t(*p - UC('0')); - if (exp_number < 0x10000000) { - exp_number = 10 * exp_number + digit; - } - ++p; - } - if (neg_exp) { - exp_number = -exp_number; - } - exponent += exp_number; - } - } else { - // If it scientific and not fixed, we have to bail out. - if ((fmt & chars_format::scientific) && !(fmt & chars_format::fixed)) { - return report_parse_error(p, parse_error::missing_exponential_part); - } - } - answer.lastmatch = p; - answer.valid = true; - - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon. - // - // We can deal with up to 19 digits. - if (digit_count > 19) { // this is uncommon - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - // We need to be mindful of the case where we only have zeroes... - // E.g., 0.000000000...000. - UC const *start = start_digits; - while ((start != pend) && (*start == UC('0') || *start == decimal_point)) { - if (*start == UC('0')) { - digit_count--; - } - start++; - } - - if (digit_count > 19) { - answer.too_many_digits = true; - // Let us start again, this time, avoiding overflows. - // We don't need to check if is_integer, since we use the - // pre-tokenized spans from above. - i = 0; - p = answer.integer.ptr; - UC const *int_end = p + answer.integer.len(); - const uint64_t minimal_nineteen_digit_integer{1000000000000000000}; - while ((i < minimal_nineteen_digit_integer) && (p != int_end)) { - i = i * 10 + uint64_t(*p - UC('0')); - ++p; - } - if (i >= minimal_nineteen_digit_integer) { // We have a big integers - exponent = end_of_integer_part - p + exp_number; - } else { // We have a value with a fractional component. - p = answer.fraction.ptr; - UC const *frac_end = p + answer.fraction.len(); - while ((i < minimal_nineteen_digit_integer) && (p != frac_end)) { - i = i * 10 + uint64_t(*p - UC('0')); - ++p; - } - exponent = answer.fraction.ptr - p + exp_number; - } - // We have now corrected both exponent and i, to a truncated value - } - } - answer.exponent = exponent; - answer.mantissa = i; - return answer; -} - -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t -parse_int_string(UC const *p, UC const *pend, T &value, int base) { - from_chars_result_t answer; - - UC const *const first = p; - - bool negative = (*p == UC('-')); - if (!std::is_signed::value && negative) { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - return answer; - } -#ifdef FASTFLOAT_ALLOWS_LEADING_PLUS // disabled by default - if ((*p == UC('-')) || (*p == UC('+'))) { -#else - if (*p == UC('-')) { -#endif - ++p; - } - - UC const *const start_num = p; - - while (p != pend && *p == UC('0')) { - ++p; - } - - const bool has_leading_zeros = p > start_num; - - UC const *const start_digits = p; - - uint64_t i = 0; - if (base == 10) { - loop_parse_if_eight_digits(p, pend, i); // use SIMD if possible - } - while (p != pend) { - uint8_t digit = ch_to_digit(*p); - if (digit >= base) { - break; - } - i = uint64_t(base) * i + digit; // might overflow, check this later - p++; - } - - size_t digit_count = size_t(p - start_digits); - - if (digit_count == 0) { - if (has_leading_zeros) { - value = 0; - answer.ec = std::errc(); - answer.ptr = p; - } else { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - } - return answer; - } - - answer.ptr = p; - - // check u64 overflow - size_t max_digits = max_digits_u64(base); - if (digit_count > max_digits) { - answer.ec = std::errc::result_out_of_range; - return answer; - } - // this check can be eliminated for all other types, but they will all require - // a max_digits(base) equivalent - if (digit_count == max_digits && i < min_safe_u64(base)) { - answer.ec = std::errc::result_out_of_range; - return answer; - } - - // check other types overflow - if (!std::is_same::value) { - if (i > uint64_t(std::numeric_limits::max()) + uint64_t(negative)) { - answer.ec = std::errc::result_out_of_range; - return answer; - } - } - - if (negative) { -#ifdef FASTFLOAT_VISUAL_STUDIO -#pragma warning(push) -#pragma warning(disable : 4146) -#endif - // this weird workaround is required because: - // - converting unsigned to signed when its value is greater than signed max - // is UB pre-C++23. - // - reinterpret_casting (~i + 1) would work, but it is not constexpr - // this is always optimized into a neg instruction (note: T is an integer - // type) - value = T(-std::numeric_limits::max() - - T(i - uint64_t(std::numeric_limits::max()))); -#ifdef FASTFLOAT_VISUAL_STUDIO -#pragma warning(pop) -#endif - } else { - value = T(i); - } - - answer.ec = std::errc(); - return answer; -} - -} // namespace fast_float - -#endif - -#ifndef FASTFLOAT_FAST_TABLE_H -#define FASTFLOAT_FAST_TABLE_H - -#include - -namespace fast_float { - -/** - * When mapping numbers from decimal to binary, - * we go from w * 10^q to m * 2^p but we have - * 10^q = 5^q * 2^q, so effectively - * we are trying to match - * w * 2^q * 5^q to m * 2^p. Thus the powers of two - * are not a concern since they can be represented - * exactly using the binary notation, only the powers of five - * affect the binary significand. - */ - -/** - * The smallest non-zero float (binary64) is 2^-1074. - * We take as input numbers of the form w x 10^q where w < 2^64. - * We have that w * 10^-343 < 2^(64-344) 5^-343 < 2^-1076. - * However, we have that - * (2^64-1) * 10^-342 = (2^64-1) * 2^-342 * 5^-342 > 2^-1074. - * Thus it is possible for a number of the form w * 10^-342 where - * w is a 64-bit value to be a non-zero floating-point number. - ********* - * Any number of form w * 10^309 where w>= 1 is going to be - * infinite in binary64 so we never need to worry about powers - * of 5 greater than 308. - */ -template struct powers_template { - - constexpr static int smallest_power_of_five = - binary_format::smallest_power_of_ten(); - constexpr static int largest_power_of_five = - binary_format::largest_power_of_ten(); - constexpr static int number_of_entries = - 2 * (largest_power_of_five - smallest_power_of_five + 1); - // Powers of five from 5^-342 all the way to 5^308 rounded toward one. - constexpr static uint64_t power_of_five_128[number_of_entries] = { - 0xeef453d6923bd65a, 0x113faa2906a13b3f, - 0x9558b4661b6565f8, 0x4ac7ca59a424c507, - 0xbaaee17fa23ebf76, 0x5d79bcf00d2df649, - 0xe95a99df8ace6f53, 0xf4d82c2c107973dc, - 0x91d8a02bb6c10594, 0x79071b9b8a4be869, - 0xb64ec836a47146f9, 0x9748e2826cdee284, - 0xe3e27a444d8d98b7, 0xfd1b1b2308169b25, - 0x8e6d8c6ab0787f72, 0xfe30f0f5e50e20f7, - 0xb208ef855c969f4f, 0xbdbd2d335e51a935, - 0xde8b2b66b3bc4723, 0xad2c788035e61382, - 0x8b16fb203055ac76, 0x4c3bcb5021afcc31, - 0xaddcb9e83c6b1793, 0xdf4abe242a1bbf3d, - 0xd953e8624b85dd78, 0xd71d6dad34a2af0d, - 0x87d4713d6f33aa6b, 0x8672648c40e5ad68, - 0xa9c98d8ccb009506, 0x680efdaf511f18c2, - 0xd43bf0effdc0ba48, 0x212bd1b2566def2, - 0x84a57695fe98746d, 0x14bb630f7604b57, - 0xa5ced43b7e3e9188, 0x419ea3bd35385e2d, - 0xcf42894a5dce35ea, 0x52064cac828675b9, - 0x818995ce7aa0e1b2, 0x7343efebd1940993, - 0xa1ebfb4219491a1f, 0x1014ebe6c5f90bf8, - 0xca66fa129f9b60a6, 0xd41a26e077774ef6, - 0xfd00b897478238d0, 0x8920b098955522b4, - 0x9e20735e8cb16382, 0x55b46e5f5d5535b0, - 0xc5a890362fddbc62, 0xeb2189f734aa831d, - 0xf712b443bbd52b7b, 0xa5e9ec7501d523e4, - 0x9a6bb0aa55653b2d, 0x47b233c92125366e, - 0xc1069cd4eabe89f8, 0x999ec0bb696e840a, - 0xf148440a256e2c76, 0xc00670ea43ca250d, - 0x96cd2a865764dbca, 0x380406926a5e5728, - 0xbc807527ed3e12bc, 0xc605083704f5ecf2, - 0xeba09271e88d976b, 0xf7864a44c633682e, - 0x93445b8731587ea3, 0x7ab3ee6afbe0211d, - 0xb8157268fdae9e4c, 0x5960ea05bad82964, - 0xe61acf033d1a45df, 0x6fb92487298e33bd, - 0x8fd0c16206306bab, 0xa5d3b6d479f8e056, - 0xb3c4f1ba87bc8696, 0x8f48a4899877186c, - 0xe0b62e2929aba83c, 0x331acdabfe94de87, - 0x8c71dcd9ba0b4925, 0x9ff0c08b7f1d0b14, - 0xaf8e5410288e1b6f, 0x7ecf0ae5ee44dd9, - 0xdb71e91432b1a24a, 0xc9e82cd9f69d6150, - 0x892731ac9faf056e, 0xbe311c083a225cd2, - 0xab70fe17c79ac6ca, 0x6dbd630a48aaf406, - 0xd64d3d9db981787d, 0x92cbbccdad5b108, - 0x85f0468293f0eb4e, 0x25bbf56008c58ea5, - 0xa76c582338ed2621, 0xaf2af2b80af6f24e, - 0xd1476e2c07286faa, 0x1af5af660db4aee1, - 0x82cca4db847945ca, 0x50d98d9fc890ed4d, - 0xa37fce126597973c, 0xe50ff107bab528a0, - 0xcc5fc196fefd7d0c, 0x1e53ed49a96272c8, - 0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7a, - 0x9faacf3df73609b1, 0x77b191618c54e9ac, - 0xc795830d75038c1d, 0xd59df5b9ef6a2417, - 0xf97ae3d0d2446f25, 0x4b0573286b44ad1d, - 0x9becce62836ac577, 0x4ee367f9430aec32, - 0xc2e801fb244576d5, 0x229c41f793cda73f, - 0xf3a20279ed56d48a, 0x6b43527578c1110f, - 0x9845418c345644d6, 0x830a13896b78aaa9, - 0xbe5691ef416bd60c, 0x23cc986bc656d553, - 0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa8, - 0x94b3a202eb1c3f39, 0x7bf7d71432f3d6a9, - 0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc53, - 0xe858ad248f5c22c9, 0xd1b3400f8f9cff68, - 0x91376c36d99995be, 0x23100809b9c21fa1, - 0xb58547448ffffb2d, 0xabd40a0c2832a78a, - 0xe2e69915b3fff9f9, 0x16c90c8f323f516c, - 0x8dd01fad907ffc3b, 0xae3da7d97f6792e3, - 0xb1442798f49ffb4a, 0x99cd11cfdf41779c, - 0xdd95317f31c7fa1d, 0x40405643d711d583, - 0x8a7d3eef7f1cfc52, 0x482835ea666b2572, - 0xad1c8eab5ee43b66, 0xda3243650005eecf, - 0xd863b256369d4a40, 0x90bed43e40076a82, - 0x873e4f75e2224e68, 0x5a7744a6e804a291, - 0xa90de3535aaae202, 0x711515d0a205cb36, - 0xd3515c2831559a83, 0xd5a5b44ca873e03, - 0x8412d9991ed58091, 0xe858790afe9486c2, - 0xa5178fff668ae0b6, 0x626e974dbe39a872, - 0xce5d73ff402d98e3, 0xfb0a3d212dc8128f, - 0x80fa687f881c7f8e, 0x7ce66634bc9d0b99, - 0xa139029f6a239f72, 0x1c1fffc1ebc44e80, - 0xc987434744ac874e, 0xa327ffb266b56220, - 0xfbe9141915d7a922, 0x4bf1ff9f0062baa8, - 0x9d71ac8fada6c9b5, 0x6f773fc3603db4a9, - 0xc4ce17b399107c22, 0xcb550fb4384d21d3, - 0xf6019da07f549b2b, 0x7e2a53a146606a48, - 0x99c102844f94e0fb, 0x2eda7444cbfc426d, - 0xc0314325637a1939, 0xfa911155fefb5308, - 0xf03d93eebc589f88, 0x793555ab7eba27ca, - 0x96267c7535b763b5, 0x4bc1558b2f3458de, - 0xbbb01b9283253ca2, 0x9eb1aaedfb016f16, - 0xea9c227723ee8bcb, 0x465e15a979c1cadc, - 0x92a1958a7675175f, 0xbfacd89ec191ec9, - 0xb749faed14125d36, 0xcef980ec671f667b, - 0xe51c79a85916f484, 0x82b7e12780e7401a, - 0x8f31cc0937ae58d2, 0xd1b2ecb8b0908810, - 0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa15, - 0xdfbdcece67006ac9, 0x67a791e093e1d49a, - 0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e0, - 0xaecc49914078536d, 0x58fae9f773886e18, - 0xda7f5bf590966848, 0xaf39a475506a899e, - 0x888f99797a5e012d, 0x6d8406c952429603, - 0xaab37fd7d8f58178, 0xc8e5087ba6d33b83, - 0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a64, - 0x855c3be0a17fcd26, 0x5cf2eea09a55067f, - 0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481e, - 0xd0601d8efc57b08b, 0xf13b94daf124da26, - 0x823c12795db6ce57, 0x76c53d08d6b70858, - 0xa2cb1717b52481ed, 0x54768c4b0c64ca6e, - 0xcb7ddcdda26da268, 0xa9942f5dcf7dfd09, - 0xfe5d54150b090b02, 0xd3f93b35435d7c4c, - 0x9efa548d26e5a6e1, 0xc47bc5014a1a6daf, - 0xc6b8e9b0709f109a, 0x359ab6419ca1091b, - 0xf867241c8cc6d4c0, 0xc30163d203c94b62, - 0x9b407691d7fc44f8, 0x79e0de63425dcf1d, - 0xc21094364dfb5636, 0x985915fc12f542e4, - 0xf294b943e17a2bc4, 0x3e6f5b7b17b2939d, - 0x979cf3ca6cec5b5a, 0xa705992ceecf9c42, - 0xbd8430bd08277231, 0x50c6ff782a838353, - 0xece53cec4a314ebd, 0xa4f8bf5635246428, - 0x940f4613ae5ed136, 0x871b7795e136be99, - 0xb913179899f68584, 0x28e2557b59846e3f, - 0xe757dd7ec07426e5, 0x331aeada2fe589cf, - 0x9096ea6f3848984f, 0x3ff0d2c85def7621, - 0xb4bca50b065abe63, 0xfed077a756b53a9, - 0xe1ebce4dc7f16dfb, 0xd3e8495912c62894, - 0x8d3360f09cf6e4bd, 0x64712dd7abbbd95c, - 0xb080392cc4349dec, 0xbd8d794d96aacfb3, - 0xdca04777f541c567, 0xecf0d7a0fc5583a0, - 0x89e42caaf9491b60, 0xf41686c49db57244, - 0xac5d37d5b79b6239, 0x311c2875c522ced5, - 0xd77485cb25823ac7, 0x7d633293366b828b, - 0x86a8d39ef77164bc, 0xae5dff9c02033197, - 0xa8530886b54dbdeb, 0xd9f57f830283fdfc, - 0xd267caa862a12d66, 0xd072df63c324fd7b, - 0x8380dea93da4bc60, 0x4247cb9e59f71e6d, - 0xa46116538d0deb78, 0x52d9be85f074e608, - 0xcd795be870516656, 0x67902e276c921f8b, - 0x806bd9714632dff6, 0xba1cd8a3db53b6, - 0xa086cfcd97bf97f3, 0x80e8a40eccd228a4, - 0xc8a883c0fdaf7df0, 0x6122cd128006b2cd, - 0xfad2a4b13d1b5d6c, 0x796b805720085f81, - 0x9cc3a6eec6311a63, 0xcbe3303674053bb0, - 0xc3f490aa77bd60fc, 0xbedbfc4411068a9c, - 0xf4f1b4d515acb93b, 0xee92fb5515482d44, - 0x991711052d8bf3c5, 0x751bdd152d4d1c4a, - 0xbf5cd54678eef0b6, 0xd262d45a78a0635d, - 0xef340a98172aace4, 0x86fb897116c87c34, - 0x9580869f0e7aac0e, 0xd45d35e6ae3d4da0, - 0xbae0a846d2195712, 0x8974836059cca109, - 0xe998d258869facd7, 0x2bd1a438703fc94b, - 0x91ff83775423cc06, 0x7b6306a34627ddcf, - 0xb67f6455292cbf08, 0x1a3bc84c17b1d542, - 0xe41f3d6a7377eeca, 0x20caba5f1d9e4a93, - 0x8e938662882af53e, 0x547eb47b7282ee9c, - 0xb23867fb2a35b28d, 0xe99e619a4f23aa43, - 0xdec681f9f4c31f31, 0x6405fa00e2ec94d4, - 0x8b3c113c38f9f37e, 0xde83bc408dd3dd04, - 0xae0b158b4738705e, 0x9624ab50b148d445, - 0xd98ddaee19068c76, 0x3badd624dd9b0957, - 0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d6, - 0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4c, - 0xd47487cc8470652b, 0x7647c3200069671f, - 0x84c8d4dfd2c63f3b, 0x29ecd9f40041e073, - 0xa5fb0a17c777cf09, 0xf468107100525890, - 0xcf79cc9db955c2cc, 0x7182148d4066eeb4, - 0x81ac1fe293d599bf, 0xc6f14cd848405530, - 0xa21727db38cb002f, 0xb8ada00e5a506a7c, - 0xca9cf1d206fdc03b, 0xa6d90811f0e4851c, - 0xfd442e4688bd304a, 0x908f4a166d1da663, - 0x9e4a9cec15763e2e, 0x9a598e4e043287fe, - 0xc5dd44271ad3cdba, 0x40eff1e1853f29fd, - 0xf7549530e188c128, 0xd12bee59e68ef47c, - 0x9a94dd3e8cf578b9, 0x82bb74f8301958ce, - 0xc13a148e3032d6e7, 0xe36a52363c1faf01, - 0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac1, - 0x96f5600f15a7b7e5, 0x29ab103a5ef8c0b9, - 0xbcb2b812db11a5de, 0x7415d448f6b6f0e7, - 0xebdf661791d60f56, 0x111b495b3464ad21, - 0x936b9fcebb25c995, 0xcab10dd900beec34, - 0xb84687c269ef3bfb, 0x3d5d514f40eea742, - 0xe65829b3046b0afa, 0xcb4a5a3112a5112, - 0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ab, - 0xb3f4e093db73a093, 0x59ed216765690f56, - 0xe0f218b8d25088b8, 0x306869c13ec3532c, - 0x8c974f7383725573, 0x1e414218c73a13fb, - 0xafbd2350644eeacf, 0xe5d1929ef90898fa, - 0xdbac6c247d62a583, 0xdf45f746b74abf39, - 0x894bc396ce5da772, 0x6b8bba8c328eb783, - 0xab9eb47c81f5114f, 0x66ea92f3f326564, - 0xd686619ba27255a2, 0xc80a537b0efefebd, - 0x8613fd0145877585, 0xbd06742ce95f5f36, - 0xa798fc4196e952e7, 0x2c48113823b73704, - 0xd17f3b51fca3a7a0, 0xf75a15862ca504c5, - 0x82ef85133de648c4, 0x9a984d73dbe722fb, - 0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebba, - 0xcc963fee10b7d1b3, 0x318df905079926a8, - 0xffbbcfe994e5c61f, 0xfdf17746497f7052, - 0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa633, - 0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc0, - 0xf9bd690a1b68637b, 0x3dfdce7aa3c673b0, - 0x9c1661a651213e2d, 0x6bea10ca65c084e, - 0xc31bfa0fe5698db8, 0x486e494fcff30a62, - 0xf3e2f893dec3f126, 0x5a89dba3c3efccfa, - 0x986ddb5c6b3a76b7, 0xf89629465a75e01c, - 0xbe89523386091465, 0xf6bbb397f1135823, - 0xee2ba6c0678b597f, 0x746aa07ded582e2c, - 0x94db483840b717ef, 0xa8c2a44eb4571cdc, - 0xba121a4650e4ddeb, 0x92f34d62616ce413, - 0xe896a0d7e51e1566, 0x77b020baf9c81d17, - 0x915e2486ef32cd60, 0xace1474dc1d122e, - 0xb5b5ada8aaff80b8, 0xd819992132456ba, - 0xe3231912d5bf60e6, 0x10e1fff697ed6c69, - 0x8df5efabc5979c8f, 0xca8d3ffa1ef463c1, - 0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb2, - 0xddd0467c64bce4a0, 0xac7cb3f6d05ddbde, - 0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96b, - 0xad4ab7112eb3929d, 0x86c16c98d2c953c6, - 0xd89d64d57a607744, 0xe871c7bf077ba8b7, - 0x87625f056c7c4a8b, 0x11471cd764ad4972, - 0xa93af6c6c79b5d2d, 0xd598e40d3dd89bcf, - 0xd389b47879823479, 0x4aff1d108d4ec2c3, - 0x843610cb4bf160cb, 0xcedf722a585139ba, - 0xa54394fe1eedb8fe, 0xc2974eb4ee658828, - 0xce947a3da6a9273e, 0x733d226229feea32, - 0x811ccc668829b887, 0x806357d5a3f525f, - 0xa163ff802a3426a8, 0xca07c2dcb0cf26f7, - 0xc9bcff6034c13052, 0xfc89b393dd02f0b5, - 0xfc2c3f3841f17c67, 0xbbac2078d443ace2, - 0x9d9ba7832936edc0, 0xd54b944b84aa4c0d, - 0xc5029163f384a931, 0xa9e795e65d4df11, - 0xf64335bcf065d37d, 0x4d4617b5ff4a16d5, - 0x99ea0196163fa42e, 0x504bced1bf8e4e45, - 0xc06481fb9bcf8d39, 0xe45ec2862f71e1d6, - 0xf07da27a82c37088, 0x5d767327bb4e5a4c, - 0x964e858c91ba2655, 0x3a6a07f8d510f86f, - 0xbbe226efb628afea, 0x890489f70a55368b, - 0xeadab0aba3b2dbe5, 0x2b45ac74ccea842e, - 0x92c8ae6b464fc96f, 0x3b0b8bc90012929d, - 0xb77ada0617e3bbcb, 0x9ce6ebb40173744, - 0xe55990879ddcaabd, 0xcc420a6a101d0515, - 0x8f57fa54c2a9eab6, 0x9fa946824a12232d, - 0xb32df8e9f3546564, 0x47939822dc96abf9, - 0xdff9772470297ebd, 0x59787e2b93bc56f7, - 0x8bfbea76c619ef36, 0x57eb4edb3c55b65a, - 0xaefae51477a06b03, 0xede622920b6b23f1, - 0xdab99e59958885c4, 0xe95fab368e45eced, - 0x88b402f7fd75539b, 0x11dbcb0218ebb414, - 0xaae103b5fcd2a881, 0xd652bdc29f26a119, - 0xd59944a37c0752a2, 0x4be76d3346f0495f, - 0x857fcae62d8493a5, 0x6f70a4400c562ddb, - 0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb952, - 0xd097ad07a71f26b2, 0x7e2000a41346a7a7, - 0x825ecc24c873782f, 0x8ed400668c0c28c8, - 0xa2f67f2dfa90563b, 0x728900802f0f32fa, - 0xcbb41ef979346bca, 0x4f2b40a03ad2ffb9, - 0xfea126b7d78186bc, 0xe2f610c84987bfa8, - 0x9f24b832e6b0f436, 0xdd9ca7d2df4d7c9, - 0xc6ede63fa05d3143, 0x91503d1c79720dbb, - 0xf8a95fcf88747d94, 0x75a44c6397ce912a, - 0x9b69dbe1b548ce7c, 0xc986afbe3ee11aba, - 0xc24452da229b021b, 0xfbe85badce996168, - 0xf2d56790ab41c2a2, 0xfae27299423fb9c3, - 0x97c560ba6b0919a5, 0xdccd879fc967d41a, - 0xbdb6b8e905cb600f, 0x5400e987bbc1c920, - 0xed246723473e3813, 0x290123e9aab23b68, - 0x9436c0760c86e30b, 0xf9a0b6720aaf6521, - 0xb94470938fa89bce, 0xf808e40e8d5b3e69, - 0xe7958cb87392c2c2, 0xb60b1d1230b20e04, - 0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c2, - 0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af3, - 0xe2280b6c20dd5232, 0x25c6da63c38de1b0, - 0x8d590723948a535f, 0x579c487e5a38ad0e, - 0xb0af48ec79ace837, 0x2d835a9df0c6d851, - 0xdcdb1b2798182244, 0xf8e431456cf88e65, - 0x8a08f0f8bf0f156b, 0x1b8e9ecb641b58ff, - 0xac8b2d36eed2dac5, 0xe272467e3d222f3f, - 0xd7adf884aa879177, 0x5b0ed81dcc6abb0f, - 0x86ccbb52ea94baea, 0x98e947129fc2b4e9, - 0xa87fea27a539e9a5, 0x3f2398d747b36224, - 0xd29fe4b18e88640e, 0x8eec7f0d19a03aad, - 0x83a3eeeef9153e89, 0x1953cf68300424ac, - 0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd7, - 0xcdb02555653131b6, 0x3792f412cb06794d, - 0x808e17555f3ebf11, 0xe2bbd88bbee40bd0, - 0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec4, - 0xc8de047564d20a8b, 0xf245825a5a445275, - 0xfb158592be068d2e, 0xeed6e2f0f0d56712, - 0x9ced737bb6c4183d, 0x55464dd69685606b, - 0xc428d05aa4751e4c, 0xaa97e14c3c26b886, - 0xf53304714d9265df, 0xd53dd99f4b3066a8, - 0x993fe2c6d07b7fab, 0xe546a8038efe4029, - 0xbf8fdb78849a5f96, 0xde98520472bdd033, - 0xef73d256a5c0f77c, 0x963e66858f6d4440, - 0x95a8637627989aad, 0xdde7001379a44aa8, - 0xbb127c53b17ec159, 0x5560c018580d5d52, - 0xe9d71b689dde71af, 0xaab8f01e6e10b4a6, - 0x9226712162ab070d, 0xcab3961304ca70e8, - 0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d22, - 0xe45c10c42a2b3b05, 0x8cb89a7db77c506a, - 0x8eb98a7a9a5b04e3, 0x77f3608e92adb242, - 0xb267ed1940f1c61c, 0x55f038b237591ed3, - 0xdf01e85f912e37a3, 0x6b6c46dec52f6688, - 0x8b61313bbabce2c6, 0x2323ac4b3b3da015, - 0xae397d8aa96c1b77, 0xabec975e0a0d081a, - 0xd9c7dced53c72255, 0x96e7bd358c904a21, - 0x881cea14545c7575, 0x7e50d64177da2e54, - 0xaa242499697392d2, 0xdde50bd1d5d0b9e9, - 0xd4ad2dbfc3d07787, 0x955e4ec64b44e864, - 0x84ec3c97da624ab4, 0xbd5af13bef0b113e, - 0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58e, - 0xcfb11ead453994ba, 0x67de18eda5814af2, - 0x81ceb32c4b43fcf4, 0x80eacf948770ced7, - 0xa2425ff75e14fc31, 0xa1258379a94d028d, - 0xcad2f7f5359a3b3e, 0x96ee45813a04330, - 0xfd87b5f28300ca0d, 0x8bca9d6e188853fc, - 0x9e74d1b791e07e48, 0x775ea264cf55347e, - 0xc612062576589dda, 0x95364afe032a819e, - 0xf79687aed3eec551, 0x3a83ddbd83f52205, - 0x9abe14cd44753b52, 0xc4926a9672793543, - 0xc16d9a0095928a27, 0x75b7053c0f178294, - 0xf1c90080baf72cb1, 0x5324c68b12dd6339, - 0x971da05074da7bee, 0xd3f6fc16ebca5e04, - 0xbce5086492111aea, 0x88f4bb1ca6bcf585, - 0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6, - 0x9392ee8e921d5d07, 0x3aff322e62439fd0, - 0xb877aa3236a4b449, 0x9befeb9fad487c3, - 0xe69594bec44de15b, 0x4c2ebe687989a9b4, - 0x901d7cf73ab0acd9, 0xf9d37014bf60a11, - 0xb424dc35095cd80f, 0x538484c19ef38c95, - 0xe12e13424bb40e13, 0x2865a5f206b06fba, - 0x8cbccc096f5088cb, 0xf93f87b7442e45d4, - 0xafebff0bcb24aafe, 0xf78f69a51539d749, - 0xdbe6fecebdedd5be, 0xb573440e5a884d1c, - 0x89705f4136b4a597, 0x31680a88f8953031, - 0xabcc77118461cefc, 0xfdc20d2b36ba7c3e, - 0xd6bf94d5e57a42bc, 0x3d32907604691b4d, - 0x8637bd05af6c69b5, 0xa63f9a49c2c1b110, - 0xa7c5ac471b478423, 0xfcf80dc33721d54, - 0xd1b71758e219652b, 0xd3c36113404ea4a9, - 0x83126e978d4fdf3b, 0x645a1cac083126ea, - 0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4, - 0xcccccccccccccccc, 0xcccccccccccccccd, - 0x8000000000000000, 0x0, - 0xa000000000000000, 0x0, - 0xc800000000000000, 0x0, - 0xfa00000000000000, 0x0, - 0x9c40000000000000, 0x0, - 0xc350000000000000, 0x0, - 0xf424000000000000, 0x0, - 0x9896800000000000, 0x0, - 0xbebc200000000000, 0x0, - 0xee6b280000000000, 0x0, - 0x9502f90000000000, 0x0, - 0xba43b74000000000, 0x0, - 0xe8d4a51000000000, 0x0, - 0x9184e72a00000000, 0x0, - 0xb5e620f480000000, 0x0, - 0xe35fa931a0000000, 0x0, - 0x8e1bc9bf04000000, 0x0, - 0xb1a2bc2ec5000000, 0x0, - 0xde0b6b3a76400000, 0x0, - 0x8ac7230489e80000, 0x0, - 0xad78ebc5ac620000, 0x0, - 0xd8d726b7177a8000, 0x0, - 0x878678326eac9000, 0x0, - 0xa968163f0a57b400, 0x0, - 0xd3c21bcecceda100, 0x0, - 0x84595161401484a0, 0x0, - 0xa56fa5b99019a5c8, 0x0, - 0xcecb8f27f4200f3a, 0x0, - 0x813f3978f8940984, 0x4000000000000000, - 0xa18f07d736b90be5, 0x5000000000000000, - 0xc9f2c9cd04674ede, 0xa400000000000000, - 0xfc6f7c4045812296, 0x4d00000000000000, - 0x9dc5ada82b70b59d, 0xf020000000000000, - 0xc5371912364ce305, 0x6c28000000000000, - 0xf684df56c3e01bc6, 0xc732000000000000, - 0x9a130b963a6c115c, 0x3c7f400000000000, - 0xc097ce7bc90715b3, 0x4b9f100000000000, - 0xf0bdc21abb48db20, 0x1e86d40000000000, - 0x96769950b50d88f4, 0x1314448000000000, - 0xbc143fa4e250eb31, 0x17d955a000000000, - 0xeb194f8e1ae525fd, 0x5dcfab0800000000, - 0x92efd1b8d0cf37be, 0x5aa1cae500000000, - 0xb7abc627050305ad, 0xf14a3d9e40000000, - 0xe596b7b0c643c719, 0x6d9ccd05d0000000, - 0x8f7e32ce7bea5c6f, 0xe4820023a2000000, - 0xb35dbf821ae4f38b, 0xdda2802c8a800000, - 0xe0352f62a19e306e, 0xd50b2037ad200000, - 0x8c213d9da502de45, 0x4526f422cc340000, - 0xaf298d050e4395d6, 0x9670b12b7f410000, - 0xdaf3f04651d47b4c, 0x3c0cdd765f114000, - 0x88d8762bf324cd0f, 0xa5880a69fb6ac800, - 0xab0e93b6efee0053, 0x8eea0d047a457a00, - 0xd5d238a4abe98068, 0x72a4904598d6d880, - 0x85a36366eb71f041, 0x47a6da2b7f864750, - 0xa70c3c40a64e6c51, 0x999090b65f67d924, - 0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d, - 0x82818f1281ed449f, 0xbff8f10e7a8921a4, - 0xa321f2d7226895c7, 0xaff72d52192b6a0d, - 0xcbea6f8ceb02bb39, 0x9bf4f8a69f764490, - 0xfee50b7025c36a08, 0x2f236d04753d5b4, - 0x9f4f2726179a2245, 0x1d762422c946590, - 0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef5, - 0xf8ebad2b84e0d58b, 0xd2e0898765a7deb2, - 0x9b934c3b330c8577, 0x63cc55f49f88eb2f, - 0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fb, - 0xf316271c7fc3908a, 0x8bef464e3945ef7a, - 0x97edd871cfda3a56, 0x97758bf0e3cbb5ac, - 0xbde94e8e43d0c8ec, 0x3d52eeed1cbea317, - 0xed63a231d4c4fb27, 0x4ca7aaa863ee4bdd, - 0x945e455f24fb1cf8, 0x8fe8caa93e74ef6a, - 0xb975d6b6ee39e436, 0xb3e2fd538e122b44, - 0xe7d34c64a9c85d44, 0x60dbbca87196b616, - 0x90e40fbeea1d3a4a, 0xbc8955e946fe31cd, - 0xb51d13aea4a488dd, 0x6babab6398bdbe41, - 0xe264589a4dcdab14, 0xc696963c7eed2dd1, - 0x8d7eb76070a08aec, 0xfc1e1de5cf543ca2, - 0xb0de65388cc8ada8, 0x3b25a55f43294bcb, - 0xdd15fe86affad912, 0x49ef0eb713f39ebe, - 0x8a2dbf142dfcc7ab, 0x6e3569326c784337, - 0xacb92ed9397bf996, 0x49c2c37f07965404, - 0xd7e77a8f87daf7fb, 0xdc33745ec97be906, - 0x86f0ac99b4e8dafd, 0x69a028bb3ded71a3, - 0xa8acd7c0222311bc, 0xc40832ea0d68ce0c, - 0xd2d80db02aabd62b, 0xf50a3fa490c30190, - 0x83c7088e1aab65db, 0x792667c6da79e0fa, - 0xa4b8cab1a1563f52, 0x577001b891185938, - 0xcde6fd5e09abcf26, 0xed4c0226b55e6f86, - 0x80b05e5ac60b6178, 0x544f8158315b05b4, - 0xa0dc75f1778e39d6, 0x696361ae3db1c721, - 0xc913936dd571c84c, 0x3bc3a19cd1e38e9, - 0xfb5878494ace3a5f, 0x4ab48a04065c723, - 0x9d174b2dcec0e47b, 0x62eb0d64283f9c76, - 0xc45d1df942711d9a, 0x3ba5d0bd324f8394, - 0xf5746577930d6500, 0xca8f44ec7ee36479, - 0x9968bf6abbe85f20, 0x7e998b13cf4e1ecb, - 0xbfc2ef456ae276e8, 0x9e3fedd8c321a67e, - 0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101e, - 0x95d04aee3b80ece5, 0xbba1f1d158724a12, - 0xbb445da9ca61281f, 0x2a8a6e45ae8edc97, - 0xea1575143cf97226, 0xf52d09d71a3293bd, - 0x924d692ca61be758, 0x593c2626705f9c56, - 0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836c, - 0xe498f455c38b997a, 0xb6dfb9c0f956447, - 0x8edf98b59a373fec, 0x4724bd4189bd5eac, - 0xb2977ee300c50fe7, 0x58edec91ec2cb657, - 0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ed, - 0x8b865b215899f46c, 0xbd79e0d20082ee74, - 0xae67f1e9aec07187, 0xecd8590680a3aa11, - 0xda01ee641a708de9, 0xe80e6f4820cc9495, - 0x884134fe908658b2, 0x3109058d147fdcdd, - 0xaa51823e34a7eede, 0xbd4b46f0599fd415, - 0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91a, - 0x850fadc09923329e, 0x3e2cf6bc604ddb0, - 0xa6539930bf6bff45, 0x84db8346b786151c, - 0xcfe87f7cef46ff16, 0xe612641865679a63, - 0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07e, - 0xa26da3999aef7749, 0xe3be5e330f38f09d, - 0xcb090c8001ab551c, 0x5cadf5bfd3072cc5, - 0xfdcb4fa002162a63, 0x73d9732fc7c8f7f6, - 0x9e9f11c4014dda7e, 0x2867e7fddcdd9afa, - 0xc646d63501a1511d, 0xb281e1fd541501b8, - 0xf7d88bc24209a565, 0x1f225a7ca91a4226, - 0x9ae757596946075f, 0x3375788de9b06958, - 0xc1a12d2fc3978937, 0x52d6b1641c83ae, - 0xf209787bb47d6b84, 0xc0678c5dbd23a49a, - 0x9745eb4d50ce6332, 0xf840b7ba963646e0, - 0xbd176620a501fbff, 0xb650e5a93bc3d898, - 0xec5d3fa8ce427aff, 0xa3e51f138ab4cebe, - 0x93ba47c980e98cdf, 0xc66f336c36b10137, - 0xb8a8d9bbe123f017, 0xb80b0047445d4184, - 0xe6d3102ad96cec1d, 0xa60dc059157491e5, - 0x9043ea1ac7e41392, 0x87c89837ad68db2f, - 0xb454e4a179dd1877, 0x29babe4598c311fb, - 0xe16a1dc9d8545e94, 0xf4296dd6fef3d67a, - 0x8ce2529e2734bb1d, 0x1899e4a65f58660c, - 0xb01ae745b101e9e4, 0x5ec05dcff72e7f8f, - 0xdc21a1171d42645d, 0x76707543f4fa1f73, - 0x899504ae72497eba, 0x6a06494a791c53a8, - 0xabfa45da0edbde69, 0x487db9d17636892, - 0xd6f8d7509292d603, 0x45a9d2845d3c42b6, - 0x865b86925b9bc5c2, 0xb8a2392ba45a9b2, - 0xa7f26836f282b732, 0x8e6cac7768d7141e, - 0xd1ef0244af2364ff, 0x3207d795430cd926, - 0x8335616aed761f1f, 0x7f44e6bd49e807b8, - 0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a6, - 0xcd036837130890a1, 0x36dba887c37a8c0f, - 0x802221226be55a64, 0xc2494954da2c9789, - 0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6c, - 0xc83553c5c8965d3d, 0x6f92829494e5acc7, - 0xfa42a8b73abbf48c, 0xcb772339ba1f17f9, - 0x9c69a97284b578d7, 0xff2a760414536efb, - 0xc38413cf25e2d70d, 0xfef5138519684aba, - 0xf46518c2ef5b8cd1, 0x7eb258665fc25d69, - 0x98bf2f79d5993802, 0xef2f773ffbd97a61, - 0xbeeefb584aff8603, 0xaafb550ffacfd8fa, - 0xeeaaba2e5dbf6784, 0x95ba2a53f983cf38, - 0x952ab45cfa97a0b2, 0xdd945a747bf26183, - 0xba756174393d88df, 0x94f971119aeef9e4, - 0xe912b9d1478ceb17, 0x7a37cd5601aab85d, - 0x91abb422ccb812ee, 0xac62e055c10ab33a, - 0xb616a12b7fe617aa, 0x577b986b314d6009, - 0xe39c49765fdf9d94, 0xed5a7e85fda0b80b, - 0x8e41ade9fbebc27d, 0x14588f13be847307, - 0xb1d219647ae6b31c, 0x596eb2d8ae258fc8, - 0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bb, - 0x8aec23d680043bee, 0x25de7bb9480d5854, - 0xada72ccc20054ae9, 0xaf561aa79a10ae6a, - 0xd910f7ff28069da4, 0x1b2ba1518094da04, - 0x87aa9aff79042286, 0x90fb44d2f05d0842, - 0xa99541bf57452b28, 0x353a1607ac744a53, - 0xd3fa922f2d1675f2, 0x42889b8997915ce8, - 0x847c9b5d7c2e09b7, 0x69956135febada11, - 0xa59bc234db398c25, 0x43fab9837e699095, - 0xcf02b2c21207ef2e, 0x94f967e45e03f4bb, - 0x8161afb94b44f57d, 0x1d1be0eebac278f5, - 0xa1ba1ba79e1632dc, 0x6462d92a69731732, - 0xca28a291859bbf93, 0x7d7b8f7503cfdcfe, - 0xfcb2cb35e702af78, 0x5cda735244c3d43e, - 0x9defbf01b061adab, 0x3a0888136afa64a7, - 0xc56baec21c7a1916, 0x88aaa1845b8fdd0, - 0xf6c69a72a3989f5b, 0x8aad549e57273d45, - 0x9a3c2087a63f6399, 0x36ac54e2f678864b, - 0xc0cb28a98fcf3c7f, 0x84576a1bb416a7dd, - 0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d5, - 0x969eb7c47859e743, 0x9f644ae5a4b1b325, - 0xbc4665b596706114, 0x873d5d9f0dde1fee, - 0xeb57ff22fc0c7959, 0xa90cb506d155a7ea, - 0x9316ff75dd87cbd8, 0x9a7f12442d588f2, - 0xb7dcbf5354e9bece, 0xc11ed6d538aeb2f, - 0xe5d3ef282a242e81, 0x8f1668c8a86da5fa, - 0x8fa475791a569d10, 0xf96e017d694487bc, - 0xb38d92d760ec4455, 0x37c981dcc395a9ac, - 0xe070f78d3927556a, 0x85bbe253f47b1417, - 0x8c469ab843b89562, 0x93956d7478ccec8e, - 0xaf58416654a6babb, 0x387ac8d1970027b2, - 0xdb2e51bfe9d0696a, 0x6997b05fcc0319e, - 0x88fcf317f22241e2, 0x441fece3bdf81f03, - 0xab3c2fddeeaad25a, 0xd527e81cad7626c3, - 0xd60b3bd56a5586f1, 0x8a71e223d8d3b074, - 0x85c7056562757456, 0xf6872d5667844e49, - 0xa738c6bebb12d16c, 0xb428f8ac016561db, - 0xd106f86e69d785c7, 0xe13336d701beba52, - 0x82a45b450226b39c, 0xecc0024661173473, - 0xa34d721642b06084, 0x27f002d7f95d0190, - 0xcc20ce9bd35c78a5, 0x31ec038df7b441f4, - 0xff290242c83396ce, 0x7e67047175a15271, - 0x9f79a169bd203e41, 0xf0062c6e984d386, - 0xc75809c42c684dd1, 0x52c07b78a3e60868, - 0xf92e0c3537826145, 0xa7709a56ccdf8a82, - 0x9bbcc7a142b17ccb, 0x88a66076400bb691, - 0xc2abf989935ddbfe, 0x6acff893d00ea435, - 0xf356f7ebf83552fe, 0x583f6b8c4124d43, - 0x98165af37b2153de, 0xc3727a337a8b704a, - 0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5c, - 0xeda2ee1c7064130c, 0x1162def06f79df73, - 0x9485d4d1c63e8be7, 0x8addcb5645ac2ba8, - 0xb9a74a0637ce2ee1, 0x6d953e2bd7173692, - 0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0437, - 0x910ab1d4db9914a0, 0x1d9c9892400a22a2, - 0xb54d5e4a127f59c8, 0x2503beb6d00cab4b, - 0xe2a0b5dc971f303a, 0x2e44ae64840fd61d, - 0x8da471a9de737e24, 0x5ceaecfed289e5d2, - 0xb10d8e1456105dad, 0x7425a83e872c5f47, - 0xdd50f1996b947518, 0xd12f124e28f77719, - 0x8a5296ffe33cc92f, 0x82bd6b70d99aaa6f, - 0xace73cbfdc0bfb7b, 0x636cc64d1001550b, - 0xd8210befd30efa5a, 0x3c47f7e05401aa4e, - 0x8714a775e3e95c78, 0x65acfaec34810a71, - 0xa8d9d1535ce3b396, 0x7f1839a741a14d0d, - 0xd31045a8341ca07c, 0x1ede48111209a050, - 0x83ea2b892091e44d, 0x934aed0aab460432, - 0xa4e4b66b68b65d60, 0xf81da84d5617853f, - 0xce1de40642e3f4b9, 0x36251260ab9d668e, - 0x80d2ae83e9ce78f3, 0xc1d72b7c6b426019, - 0xa1075a24e4421730, 0xb24cf65b8612f81f, - 0xc94930ae1d529cfc, 0xdee033f26797b627, - 0xfb9b7cd9a4a7443c, 0x169840ef017da3b1, - 0x9d412e0806e88aa5, 0x8e1f289560ee864e, - 0xc491798a08a2ad4e, 0xf1a6f2bab92a27e2, - 0xf5b5d7ec8acb58a2, 0xae10af696774b1db, - 0x9991a6f3d6bf1765, 0xacca6da1e0a8ef29, - 0xbff610b0cc6edd3f, 0x17fd090a58d32af3, - 0xeff394dcff8a948e, 0xddfc4b4cef07f5b0, - 0x95f83d0a1fb69cd9, 0x4abdaf101564f98e, - 0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f1, - 0xea53df5fd18d5513, 0x84c86189216dc5ed, - 0x92746b9be2f8552c, 0x32fd3cf5b4e49bb4, - 0xb7118682dbb66a77, 0x3fbc8c33221dc2a1, - 0xe4d5e82392a40515, 0xfabaf3feaa5334a, - 0x8f05b1163ba6832d, 0x29cb4d87f2a7400e, - 0xb2c71d5bca9023f8, 0x743e20e9ef511012, - 0xdf78e4b2bd342cf6, 0x914da9246b255416, - 0x8bab8eefb6409c1a, 0x1ad089b6c2f7548e, - 0xae9672aba3d0c320, 0xa184ac2473b529b1, - 0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741e, - 0x8865899617fb1871, 0x7e2fa67c7a658892, - 0xaa7eebfb9df9de8d, 0xddbb901b98feeab7, - 0xd51ea6fa85785631, 0x552a74227f3ea565, - 0x8533285c936b35de, 0xd53a88958f87275f, - 0xa67ff273b8460356, 0x8a892abaf368f137, - 0xd01fef10a657842c, 0x2d2b7569b0432d85, - 0x8213f56a67f6b29b, 0x9c3b29620e29fc73, - 0xa298f2c501f45f42, 0x8349f3ba91b47b8f, - 0xcb3f2f7642717713, 0x241c70a936219a73, - 0xfe0efb53d30dd4d7, 0xed238cd383aa0110, - 0x9ec95d1463e8a506, 0xf4363804324a40aa, - 0xc67bb4597ce2ce48, 0xb143c6053edcd0d5, - 0xf81aa16fdc1b81da, 0xdd94b7868e94050a, - 0x9b10a4e5e9913128, 0xca7cf2b4191c8326, - 0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f0, - 0xf24a01a73cf2dccf, 0xbc633b39673c8cec, - 0x976e41088617ca01, 0xd5be0503e085d813, - 0xbd49d14aa79dbc82, 0x4b2d8644d8a74e18, - 0xec9c459d51852ba2, 0xddf8e7d60ed1219e, - 0x93e1ab8252f33b45, 0xcabb90e5c942b503, - 0xb8da1662e7b00a17, 0x3d6a751f3b936243, - 0xe7109bfba19c0c9d, 0xcc512670a783ad4, - 0x906a617d450187e2, 0x27fb2b80668b24c5, - 0xb484f9dc9641e9da, 0xb1f9f660802dedf6, - 0xe1a63853bbd26451, 0x5e7873f8a0396973, - 0x8d07e33455637eb2, 0xdb0b487b6423e1e8, - 0xb049dc016abc5e5f, 0x91ce1a9a3d2cda62, - 0xdc5c5301c56b75f7, 0x7641a140cc7810fb, - 0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9d, - 0xac2820d9623bf429, 0x546345fa9fbdcd44, - 0xd732290fbacaf133, 0xa97c177947ad4095, - 0x867f59a9d4bed6c0, 0x49ed8eabcccc485d, - 0xa81f301449ee8c70, 0x5c68f256bfff5a74, - 0xd226fc195c6a2f8c, 0x73832eec6fff3111, - 0x83585d8fd9c25db7, 0xc831fd53c5ff7eab, - 0xa42e74f3d032f525, 0xba3e7ca8b77f5e55, - 0xcd3a1230c43fb26f, 0x28ce1bd2e55f35eb, - 0x80444b5e7aa7cf85, 0x7980d163cf5b81b3, - 0xa0555e361951c366, 0xd7e105bcc332621f, - 0xc86ab5c39fa63440, 0x8dd9472bf3fefaa7, - 0xfa856334878fc150, 0xb14f98f6f0feb951, - 0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d3, - 0xc3b8358109e84f07, 0xa862f80ec4700c8, - 0xf4a642e14c6262c8, 0xcd27bb612758c0fa, - 0x98e7e9cccfbd7dbd, 0x8038d51cb897789c, - 0xbf21e44003acdd2c, 0xe0470a63e6bd56c3, - 0xeeea5d5004981478, 0x1858ccfce06cac74, - 0x95527a5202df0ccb, 0xf37801e0c43ebc8, - 0xbaa718e68396cffd, 0xd30560258f54e6ba, - 0xe950df20247c83fd, 0x47c6b82ef32a2069, - 0x91d28b7416cdd27e, 0x4cdc331d57fa5441, - 0xb6472e511c81471d, 0xe0133fe4adf8e952, - 0xe3d8f9e563a198e5, 0x58180fddd97723a6, - 0x8e679c2f5e44ff8f, 0x570f09eaa7ea7648, - }; -}; - -template -constexpr uint64_t - powers_template::power_of_five_128[number_of_entries]; - -using powers = powers_template<>; - -} // namespace fast_float - -#endif - -#ifndef FASTFLOAT_DECIMAL_TO_BINARY_H -#define FASTFLOAT_DECIMAL_TO_BINARY_H - -#include -#include -#include -#include -#include -#include - -namespace fast_float { - -// This will compute or rather approximate w * 5**q and return a pair of 64-bit -// words approximating the result, with the "high" part corresponding to the -// most significant bits and the low part corresponding to the least significant -// bits. -// -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 value128 -compute_product_approximation(int64_t q, uint64_t w) { - const int index = 2 * int(q - powers::smallest_power_of_five); - // For small values of q, e.g., q in [0,27], the answer is always exact - // because The line value128 firstproduct = full_multiplication(w, - // power_of_five_128[index]); gives the exact answer. - value128 firstproduct = - full_multiplication(w, powers::power_of_five_128[index]); - static_assert((bit_precision >= 0) && (bit_precision <= 64), - " precision should be in (0,64]"); - constexpr uint64_t precision_mask = - (bit_precision < 64) ? (uint64_t(0xFFFFFFFFFFFFFFFF) >> bit_precision) - : uint64_t(0xFFFFFFFFFFFFFFFF); - if ((firstproduct.high & precision_mask) == - precision_mask) { // could further guard with (lower + w < lower) - // regarding the second product, we only need secondproduct.high, but our - // expectation is that the compiler will optimize this extra work away if - // needed. - value128 secondproduct = - full_multiplication(w, powers::power_of_five_128[index + 1]); - firstproduct.low += secondproduct.high; - if (secondproduct.high > firstproduct.low) { - firstproduct.high++; - } - } - return firstproduct; -} - -namespace detail { -/** - * For q in (0,350), we have that - * f = (((152170 + 65536) * q ) >> 16); - * is equal to - * floor(p) + q - * where - * p = log(5**q)/log(2) = q * log(5)/log(2) - * - * For negative values of q in (-400,0), we have that - * f = (((152170 + 65536) * q ) >> 16); - * is equal to - * -ceil(p) + q - * where - * p = log(5**-q)/log(2) = -q * log(5)/log(2) - */ -constexpr fastfloat_really_inline int32_t power(int32_t q) noexcept { - return (((152170 + 65536) * q) >> 16) + 63; -} -} // namespace detail - -// create an adjusted mantissa, biased by the invalid power2 -// for significant digits already multiplied by 10 ** q. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 adjusted_mantissa -compute_error_scaled(int64_t q, uint64_t w, int lz) noexcept { - int hilz = int(w >> 63) ^ 1; - adjusted_mantissa answer; - answer.mantissa = w << hilz; - int bias = binary::mantissa_explicit_bits() - binary::minimum_exponent(); - answer.power2 = int32_t(detail::power(int32_t(q)) + bias - hilz - lz - 62 + - invalid_am_bias); - return answer; -} - -// w * 10 ** q, without rounding the representation up. -// the power2 in the exponent will be adjusted by invalid_am_bias. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa -compute_error(int64_t q, uint64_t w) noexcept { - int lz = leading_zeroes(w); - w <<= lz; - value128 product = - compute_product_approximation(q, w); - return compute_error_scaled(q, product.high, lz); -} - -// w * 10 ** q -// The returned value should be a valid ieee64 number that simply need to be -// packed. However, in some very rare cases, the computation will fail. In such -// cases, we return an adjusted_mantissa with a negative power of 2: the caller -// should recompute in such cases. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa -compute_float(int64_t q, uint64_t w) noexcept { - adjusted_mantissa answer; - if ((w == 0) || (q < binary::smallest_power_of_ten())) { - answer.power2 = 0; - answer.mantissa = 0; - // result should be zero - return answer; - } - if (q > binary::largest_power_of_ten()) { - // we want to get infinity: - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - return answer; - } - // At this point in time q is in [powers::smallest_power_of_five, - // powers::largest_power_of_five]. - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(w); - w <<= lz; - - // The required precision is binary::mantissa_explicit_bits() + 3 because - // 1. We need the implicit bit - // 2. We need an extra bit for rounding purposes - // 3. We might lose a bit due to the "upperbit" routine (result too small, - // requiring a shift) - - value128 product = - compute_product_approximation(q, w); - // The computed 'product' is always sufficient. - // Mathematical proof: - // Noble Mushtak and Daniel Lemire, Fast Number Parsing Without Fallback (to - // appear) See script/mushtak_lemire.py - - // The "compute_product_approximation" function can be slightly slower than a - // branchless approach: value128 product = compute_product(q, w); but in - // practice, we can win big with the compute_product_approximation if its - // additional branch is easily predicted. Which is best is data specific. - int upperbit = int(product.high >> 63); - int shift = upperbit + 64 - binary::mantissa_explicit_bits() - 3; - - answer.mantissa = product.high >> shift; - - answer.power2 = int32_t(detail::power(int32_t(q)) + upperbit - lz - - binary::minimum_exponent()); - if (answer.power2 <= 0) { // we have a subnormal? - // Here have that answer.power2 <= 0 so -answer.power2 >= 0 - if (-answer.power2 + 1 >= - 64) { // if we have more than 64 bits below the minimum exponent, you - // have a zero for sure. - answer.power2 = 0; - answer.mantissa = 0; - // result should be zero - return answer; - } - // next line is safe because -answer.power2 + 1 < 64 - answer.mantissa >>= -answer.power2 + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - answer.mantissa += (answer.mantissa & 1); // round up - answer.mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - answer.power2 = - (answer.mantissa < (uint64_t(1) << binary::mantissa_explicit_bits())) - ? 0 - : 1; - return answer; - } - - // usually, we round *up*, but if we fall right in between and and we have an - // even basis, we need to round down - // We are only concerned with the cases where 5**q fits in single 64-bit word. - if ((product.low <= 1) && (q >= binary::min_exponent_round_to_even()) && - (q <= binary::max_exponent_round_to_even()) && - ((answer.mantissa & 3) == 1)) { // we may fall between two floats! - // To be in-between two floats we need that in doing - // answer.mantissa = product.high >> (upperbit + 64 - - // binary::mantissa_explicit_bits() - 3); - // ... we dropped out only zeroes. But if this happened, then we can go - // back!!! - if ((answer.mantissa << shift) == product.high) { - answer.mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - answer.mantissa += (answer.mantissa & 1); // round up - answer.mantissa >>= 1; - if (answer.mantissa >= (uint64_t(2) << binary::mantissa_explicit_bits())) { - answer.mantissa = (uint64_t(1) << binary::mantissa_explicit_bits()); - answer.power2++; // undo previous addition - } - - answer.mantissa &= ~(uint64_t(1) << binary::mantissa_explicit_bits()); - if (answer.power2 >= binary::infinite_power()) { // infinity - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - } - return answer; -} - -} // namespace fast_float - -#endif - -#ifndef FASTFLOAT_BIGINT_H -#define FASTFLOAT_BIGINT_H - -#include -#include -#include -#include - - -namespace fast_float { - -// the limb width: we want efficient multiplication of double the bits in -// limb, or for 64-bit limbs, at least 64-bit multiplication where we can -// extract the high and low parts efficiently. this is every 64-bit -// architecture except for sparc, which emulates 128-bit multiplication. -// we might have platforms where `CHAR_BIT` is not 8, so let's avoid -// doing `8 * sizeof(limb)`. -#if defined(FASTFLOAT_64BIT) && !defined(__sparc) -#define FASTFLOAT_64BIT_LIMB 1 -typedef uint64_t limb; -constexpr size_t limb_bits = 64; -#else -#define FASTFLOAT_32BIT_LIMB -typedef uint32_t limb; -constexpr size_t limb_bits = 32; -#endif - -typedef span limb_span; - -// number of bits in a bigint. this needs to be at least the number -// of bits required to store the largest bigint, which is -// `log2(10**(digits + max_exp))`, or `log2(10**(767 + 342))`, or -// ~3600 bits, so we round to 4000. -constexpr size_t bigint_bits = 4000; -constexpr size_t bigint_limbs = bigint_bits / limb_bits; - -// vector-like type that is allocated on the stack. the entire -// buffer is pre-allocated, and only the length changes. -template struct stackvec { - limb data[size]; - // we never need more than 150 limbs - uint16_t length{0}; - - stackvec() = default; - stackvec(const stackvec &) = delete; - stackvec &operator=(const stackvec &) = delete; - stackvec(stackvec &&) = delete; - stackvec &operator=(stackvec &&other) = delete; - - // create stack vector from existing limb span. - FASTFLOAT_CONSTEXPR20 stackvec(limb_span s) { - FASTFLOAT_ASSERT(try_extend(s)); - } - - FASTFLOAT_CONSTEXPR14 limb &operator[](size_t index) noexcept { - FASTFLOAT_DEBUG_ASSERT(index < length); - return data[index]; - } - FASTFLOAT_CONSTEXPR14 const limb &operator[](size_t index) const noexcept { - FASTFLOAT_DEBUG_ASSERT(index < length); - return data[index]; - } - // index from the end of the container - FASTFLOAT_CONSTEXPR14 const limb &rindex(size_t index) const noexcept { - FASTFLOAT_DEBUG_ASSERT(index < length); - size_t rindex = length - index - 1; - return data[rindex]; - } - - // set the length, without bounds checking. - FASTFLOAT_CONSTEXPR14 void set_len(size_t len) noexcept { - length = uint16_t(len); - } - constexpr size_t len() const noexcept { return length; } - constexpr bool is_empty() const noexcept { return length == 0; } - constexpr size_t capacity() const noexcept { return size; } - // append item to vector, without bounds checking - FASTFLOAT_CONSTEXPR14 void push_unchecked(limb value) noexcept { - data[length] = value; - length++; - } - // append item to vector, returning if item was added - FASTFLOAT_CONSTEXPR14 bool try_push(limb value) noexcept { - if (len() < capacity()) { - push_unchecked(value); - return true; - } else { - return false; - } - } - // add items to the vector, from a span, without bounds checking - FASTFLOAT_CONSTEXPR20 void extend_unchecked(limb_span s) noexcept { - limb *ptr = data + length; - std::copy_n(s.ptr, s.len(), ptr); - set_len(len() + s.len()); - } - // try to add items to the vector, returning if items were added - FASTFLOAT_CONSTEXPR20 bool try_extend(limb_span s) noexcept { - if (len() + s.len() <= capacity()) { - extend_unchecked(s); - return true; - } else { - return false; - } - } - // resize the vector, without bounds checking - // if the new size is longer than the vector, assign value to each - // appended item. - FASTFLOAT_CONSTEXPR20 - void resize_unchecked(size_t new_len, limb value) noexcept { - if (new_len > len()) { - size_t count = new_len - len(); - limb *first = data + len(); - limb *last = first + count; - ::std::fill(first, last, value); - set_len(new_len); - } else { - set_len(new_len); - } - } - // try to resize the vector, returning if the vector was resized. - FASTFLOAT_CONSTEXPR20 bool try_resize(size_t new_len, limb value) noexcept { - if (new_len > capacity()) { - return false; - } else { - resize_unchecked(new_len, value); - return true; - } - } - // check if any limbs are non-zero after the given index. - // this needs to be done in reverse order, since the index - // is relative to the most significant limbs. - FASTFLOAT_CONSTEXPR14 bool nonzero(size_t index) const noexcept { - while (index < len()) { - if (rindex(index) != 0) { - return true; - } - index++; - } - return false; - } - // normalize the big integer, so most-significant zero limbs are removed. - FASTFLOAT_CONSTEXPR14 void normalize() noexcept { - while (len() > 0 && rindex(0) == 0) { - length--; - } - } -}; - -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint64_t -empty_hi64(bool &truncated) noexcept { - truncated = false; - return 0; -} - -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t -uint64_hi64(uint64_t r0, bool &truncated) noexcept { - truncated = false; - int shl = leading_zeroes(r0); - return r0 << shl; -} - -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t -uint64_hi64(uint64_t r0, uint64_t r1, bool &truncated) noexcept { - int shl = leading_zeroes(r0); - if (shl == 0) { - truncated = r1 != 0; - return r0; - } else { - int shr = 64 - shl; - truncated = (r1 << shl) != 0; - return (r0 << shl) | (r1 >> shr); - } -} - -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t -uint32_hi64(uint32_t r0, bool &truncated) noexcept { - return uint64_hi64(r0, truncated); -} - -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t -uint32_hi64(uint32_t r0, uint32_t r1, bool &truncated) noexcept { - uint64_t x0 = r0; - uint64_t x1 = r1; - return uint64_hi64((x0 << 32) | x1, truncated); -} - -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t -uint32_hi64(uint32_t r0, uint32_t r1, uint32_t r2, bool &truncated) noexcept { - uint64_t x0 = r0; - uint64_t x1 = r1; - uint64_t x2 = r2; - return uint64_hi64(x0, (x1 << 32) | x2, truncated); -} - -// add two small integers, checking for overflow. -// we want an efficient operation. for msvc, where -// we don't have built-in intrinsics, this is still -// pretty fast. -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 limb -scalar_add(limb x, limb y, bool &overflow) noexcept { - limb z; -// gcc and clang -#if defined(__has_builtin) -#if __has_builtin(__builtin_add_overflow) - if (!cpp20_and_in_constexpr()) { - overflow = __builtin_add_overflow(x, y, &z); - return z; - } -#endif -#endif - - // generic, this still optimizes correctly on MSVC. - z = x + y; - overflow = z < x; - return z; -} - -// multiply two small integers, getting both the high and low bits. -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 limb -scalar_mul(limb x, limb y, limb &carry) noexcept { -#ifdef FASTFLOAT_64BIT_LIMB -#if defined(__SIZEOF_INT128__) - // GCC and clang both define it as an extension. - __uint128_t z = __uint128_t(x) * __uint128_t(y) + __uint128_t(carry); - carry = limb(z >> limb_bits); - return limb(z); -#else - // fallback, no native 128-bit integer multiplication with carry. - // on msvc, this optimizes identically, somehow. - value128 z = full_multiplication(x, y); - bool overflow; - z.low = scalar_add(z.low, carry, overflow); - z.high += uint64_t(overflow); // cannot overflow - carry = z.high; - return z.low; -#endif -#else - uint64_t z = uint64_t(x) * uint64_t(y) + uint64_t(carry); - carry = limb(z >> limb_bits); - return limb(z); -#endif -} - -// add scalar value to bigint starting from offset. -// used in grade school multiplication -template -inline FASTFLOAT_CONSTEXPR20 bool small_add_from(stackvec &vec, limb y, - size_t start) noexcept { - size_t index = start; - limb carry = y; - bool overflow; - while (carry != 0 && index < vec.len()) { - vec[index] = scalar_add(vec[index], carry, overflow); - carry = limb(overflow); - index += 1; - } - if (carry != 0) { - FASTFLOAT_TRY(vec.try_push(carry)); - } - return true; -} - -// add scalar value to bigint. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool -small_add(stackvec &vec, limb y) noexcept { - return small_add_from(vec, y, 0); -} - -// multiply bigint by scalar value. -template -inline FASTFLOAT_CONSTEXPR20 bool small_mul(stackvec &vec, - limb y) noexcept { - limb carry = 0; - for (size_t index = 0; index < vec.len(); index++) { - vec[index] = scalar_mul(vec[index], y, carry); - } - if (carry != 0) { - FASTFLOAT_TRY(vec.try_push(carry)); - } - return true; -} - -// add bigint to bigint starting from index. -// used in grade school multiplication -template -FASTFLOAT_CONSTEXPR20 bool large_add_from(stackvec &x, limb_span y, - size_t start) noexcept { - // the effective x buffer is from `xstart..x.len()`, so exit early - // if we can't get that current range. - if (x.len() < start || y.len() > x.len() - start) { - FASTFLOAT_TRY(x.try_resize(y.len() + start, 0)); - } - - bool carry = false; - for (size_t index = 0; index < y.len(); index++) { - limb xi = x[index + start]; - limb yi = y[index]; - bool c1 = false; - bool c2 = false; - xi = scalar_add(xi, yi, c1); - if (carry) { - xi = scalar_add(xi, 1, c2); - } - x[index + start] = xi; - carry = c1 | c2; - } - - // handle overflow - if (carry) { - FASTFLOAT_TRY(small_add_from(x, 1, y.len() + start)); - } - return true; -} - -// add bigint to bigint. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool -large_add_from(stackvec &x, limb_span y) noexcept { - return large_add_from(x, y, 0); -} - -// grade-school multiplication algorithm -template -FASTFLOAT_CONSTEXPR20 bool long_mul(stackvec &x, limb_span y) noexcept { - limb_span xs = limb_span(x.data, x.len()); - stackvec z(xs); - limb_span zs = limb_span(z.data, z.len()); - - if (y.len() != 0) { - limb y0 = y[0]; - FASTFLOAT_TRY(small_mul(x, y0)); - for (size_t index = 1; index < y.len(); index++) { - limb yi = y[index]; - stackvec zi; - if (yi != 0) { - // re-use the same buffer throughout - zi.set_len(0); - FASTFLOAT_TRY(zi.try_extend(zs)); - FASTFLOAT_TRY(small_mul(zi, yi)); - limb_span zis = limb_span(zi.data, zi.len()); - FASTFLOAT_TRY(large_add_from(x, zis, index)); - } - } - } - - x.normalize(); - return true; -} - -// grade-school multiplication algorithm -template -FASTFLOAT_CONSTEXPR20 bool large_mul(stackvec &x, limb_span y) noexcept { - if (y.len() == 1) { - FASTFLOAT_TRY(small_mul(x, y[0])); - } else { - FASTFLOAT_TRY(long_mul(x, y)); - } - return true; -} - -template struct pow5_tables { - static constexpr uint32_t large_step = 135; - static constexpr uint64_t small_power_of_5[] = { - 1UL, - 5UL, - 25UL, - 125UL, - 625UL, - 3125UL, - 15625UL, - 78125UL, - 390625UL, - 1953125UL, - 9765625UL, - 48828125UL, - 244140625UL, - 1220703125UL, - 6103515625UL, - 30517578125UL, - 152587890625UL, - 762939453125UL, - 3814697265625UL, - 19073486328125UL, - 95367431640625UL, - 476837158203125UL, - 2384185791015625UL, - 11920928955078125UL, - 59604644775390625UL, - 298023223876953125UL, - 1490116119384765625UL, - 7450580596923828125UL, - }; -#ifdef FASTFLOAT_64BIT_LIMB - constexpr static limb large_power_of_5[] = { - 1414648277510068013UL, 9180637584431281687UL, 4539964771860779200UL, - 10482974169319127550UL, 198276706040285095UL}; -#else - constexpr static limb large_power_of_5[] = { - 4279965485U, 329373468U, 4020270615U, 2137533757U, 4287402176U, - 1057042919U, 1071430142U, 2440757623U, 381945767U, 46164893U}; -#endif -}; - -template constexpr uint32_t pow5_tables::large_step; - -template constexpr uint64_t pow5_tables::small_power_of_5[]; - -template constexpr limb pow5_tables::large_power_of_5[]; - -// big integer type. implements a small subset of big integer -// arithmetic, using simple algorithms since asymptotically -// faster algorithms are slower for a small number of limbs. -// all operations assume the big-integer is normalized. -struct bigint : pow5_tables<> { - // storage of the limbs, in little-endian order. - stackvec vec; - - FASTFLOAT_CONSTEXPR20 bigint() : vec() {} - bigint(const bigint &) = delete; - bigint &operator=(const bigint &) = delete; - bigint(bigint &&) = delete; - bigint &operator=(bigint &&other) = delete; - - FASTFLOAT_CONSTEXPR20 bigint(uint64_t value) : vec() { -#ifdef FASTFLOAT_64BIT_LIMB - vec.push_unchecked(value); -#else - vec.push_unchecked(uint32_t(value)); - vec.push_unchecked(uint32_t(value >> 32)); -#endif - vec.normalize(); - } - - // get the high 64 bits from the vector, and if bits were truncated. - // this is to get the significant digits for the float. - FASTFLOAT_CONSTEXPR20 uint64_t hi64(bool &truncated) const noexcept { -#ifdef FASTFLOAT_64BIT_LIMB - if (vec.len() == 0) { - return empty_hi64(truncated); - } else if (vec.len() == 1) { - return uint64_hi64(vec.rindex(0), truncated); - } else { - uint64_t result = uint64_hi64(vec.rindex(0), vec.rindex(1), truncated); - truncated |= vec.nonzero(2); - return result; - } -#else - if (vec.len() == 0) { - return empty_hi64(truncated); - } else if (vec.len() == 1) { - return uint32_hi64(vec.rindex(0), truncated); - } else if (vec.len() == 2) { - return uint32_hi64(vec.rindex(0), vec.rindex(1), truncated); - } else { - uint64_t result = - uint32_hi64(vec.rindex(0), vec.rindex(1), vec.rindex(2), truncated); - truncated |= vec.nonzero(3); - return result; - } -#endif - } - - // compare two big integers, returning the large value. - // assumes both are normalized. if the return value is - // negative, other is larger, if the return value is - // positive, this is larger, otherwise they are equal. - // the limbs are stored in little-endian order, so we - // must compare the limbs in ever order. - FASTFLOAT_CONSTEXPR20 int compare(const bigint &other) const noexcept { - if (vec.len() > other.vec.len()) { - return 1; - } else if (vec.len() < other.vec.len()) { - return -1; - } else { - for (size_t index = vec.len(); index > 0; index--) { - limb xi = vec[index - 1]; - limb yi = other.vec[index - 1]; - if (xi > yi) { - return 1; - } else if (xi < yi) { - return -1; - } - } - return 0; - } - } - - // shift left each limb n bits, carrying over to the new limb - // returns true if we were able to shift all the digits. - FASTFLOAT_CONSTEXPR20 bool shl_bits(size_t n) noexcept { - // Internally, for each item, we shift left by n, and add the previous - // right shifted limb-bits. - // For example, we transform (for u8) shifted left 2, to: - // b10100100 b01000010 - // b10 b10010001 b00001000 - FASTFLOAT_DEBUG_ASSERT(n != 0); - FASTFLOAT_DEBUG_ASSERT(n < sizeof(limb) * 8); - - size_t shl = n; - size_t shr = limb_bits - shl; - limb prev = 0; - for (size_t index = 0; index < vec.len(); index++) { - limb xi = vec[index]; - vec[index] = (xi << shl) | (prev >> shr); - prev = xi; - } - - limb carry = prev >> shr; - if (carry != 0) { - return vec.try_push(carry); - } - return true; - } - - // move the limbs left by `n` limbs. - FASTFLOAT_CONSTEXPR20 bool shl_limbs(size_t n) noexcept { - FASTFLOAT_DEBUG_ASSERT(n != 0); - if (n + vec.len() > vec.capacity()) { - return false; - } else if (!vec.is_empty()) { - // move limbs - limb *dst = vec.data + n; - const limb *src = vec.data; - std::copy_backward(src, src + vec.len(), dst + vec.len()); - // fill in empty limbs - limb *first = vec.data; - limb *last = first + n; - ::std::fill(first, last, 0); - vec.set_len(n + vec.len()); - return true; - } else { - return true; - } - } - - // move the limbs left by `n` bits. - FASTFLOAT_CONSTEXPR20 bool shl(size_t n) noexcept { - size_t rem = n % limb_bits; - size_t div = n / limb_bits; - if (rem != 0) { - FASTFLOAT_TRY(shl_bits(rem)); - } - if (div != 0) { - FASTFLOAT_TRY(shl_limbs(div)); - } - return true; - } - - // get the number of leading zeros in the bigint. - FASTFLOAT_CONSTEXPR20 int ctlz() const noexcept { - if (vec.is_empty()) { - return 0; - } else { -#ifdef FASTFLOAT_64BIT_LIMB - return leading_zeroes(vec.rindex(0)); -#else - // no use defining a specialized leading_zeroes for a 32-bit type. - uint64_t r0 = vec.rindex(0); - return leading_zeroes(r0 << 32); -#endif - } - } - - // get the number of bits in the bigint. - FASTFLOAT_CONSTEXPR20 int bit_length() const noexcept { - int lz = ctlz(); - return int(limb_bits * vec.len()) - lz; - } - - FASTFLOAT_CONSTEXPR20 bool mul(limb y) noexcept { return small_mul(vec, y); } - - FASTFLOAT_CONSTEXPR20 bool add(limb y) noexcept { return small_add(vec, y); } - - // multiply as if by 2 raised to a power. - FASTFLOAT_CONSTEXPR20 bool pow2(uint32_t exp) noexcept { return shl(exp); } - - // multiply as if by 5 raised to a power. - FASTFLOAT_CONSTEXPR20 bool pow5(uint32_t exp) noexcept { - // multiply by a power of 5 - size_t large_length = sizeof(large_power_of_5) / sizeof(limb); - limb_span large = limb_span(large_power_of_5, large_length); - while (exp >= large_step) { - FASTFLOAT_TRY(large_mul(vec, large)); - exp -= large_step; - } -#ifdef FASTFLOAT_64BIT_LIMB - uint32_t small_step = 27; - limb max_native = 7450580596923828125UL; -#else - uint32_t small_step = 13; - limb max_native = 1220703125U; -#endif - while (exp >= small_step) { - FASTFLOAT_TRY(small_mul(vec, max_native)); - exp -= small_step; - } - if (exp != 0) { - // Work around clang bug https://godbolt.org/z/zedh7rrhc - // This is similar to https://github.com/llvm/llvm-project/issues/47746, - // except the workaround described there don't work here - FASTFLOAT_TRY(small_mul( - vec, limb(((void)small_power_of_5[0], small_power_of_5[exp])))); - } - - return true; - } - - // multiply as if by 10 raised to a power. - FASTFLOAT_CONSTEXPR20 bool pow10(uint32_t exp) noexcept { - FASTFLOAT_TRY(pow5(exp)); - return pow2(exp); - } -}; - -} // namespace fast_float - -#endif - -#ifndef FASTFLOAT_DIGIT_COMPARISON_H -#define FASTFLOAT_DIGIT_COMPARISON_H - -#include -#include -#include -#include - - -namespace fast_float { - -// 1e0 to 1e19 -constexpr static uint64_t powers_of_ten_uint64[] = {1UL, - 10UL, - 100UL, - 1000UL, - 10000UL, - 100000UL, - 1000000UL, - 10000000UL, - 100000000UL, - 1000000000UL, - 10000000000UL, - 100000000000UL, - 1000000000000UL, - 10000000000000UL, - 100000000000000UL, - 1000000000000000UL, - 10000000000000000UL, - 100000000000000000UL, - 1000000000000000000UL, - 10000000000000000000UL}; - -// calculate the exponent, in scientific notation, of the number. -// this algorithm is not even close to optimized, but it has no practical -// effect on performance: in order to have a faster algorithm, we'd need -// to slow down performance for faster algorithms, and this is still fast. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 int32_t -scientific_exponent(parsed_number_string_t &num) noexcept { - uint64_t mantissa = num.mantissa; - int32_t exponent = int32_t(num.exponent); - while (mantissa >= 10000) { - mantissa /= 10000; - exponent += 4; - } - while (mantissa >= 100) { - mantissa /= 100; - exponent += 2; - } - while (mantissa >= 10) { - mantissa /= 10; - exponent += 1; - } - return exponent; -} - -// this converts a native floating-point number to an extended-precision float. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa -to_extended(T value) noexcept { - using equiv_uint = typename binary_format::equiv_uint; - constexpr equiv_uint exponent_mask = binary_format::exponent_mask(); - constexpr equiv_uint mantissa_mask = binary_format::mantissa_mask(); - constexpr equiv_uint hidden_bit_mask = binary_format::hidden_bit_mask(); - - adjusted_mantissa am; - int32_t bias = binary_format::mantissa_explicit_bits() - - binary_format::minimum_exponent(); - equiv_uint bits; -#if FASTFLOAT_HAS_BIT_CAST - bits = std::bit_cast(value); -#else - ::memcpy(&bits, &value, sizeof(T)); -#endif - if ((bits & exponent_mask) == 0) { - // denormal - am.power2 = 1 - bias; - am.mantissa = bits & mantissa_mask; - } else { - // normal - am.power2 = int32_t((bits & exponent_mask) >> - binary_format::mantissa_explicit_bits()); - am.power2 -= bias; - am.mantissa = (bits & mantissa_mask) | hidden_bit_mask; - } - - return am; -} - -// get the extended precision value of the halfway point between b and b+u. -// we are given a native float that represents b, so we need to adjust it -// halfway between b and b+u. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa -to_extended_halfway(T value) noexcept { - adjusted_mantissa am = to_extended(value); - am.mantissa <<= 1; - am.mantissa += 1; - am.power2 -= 1; - return am; -} - -// round an extended-precision float to the nearest machine float. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void round(adjusted_mantissa &am, - callback cb) noexcept { - int32_t mantissa_shift = 64 - binary_format::mantissa_explicit_bits() - 1; - if (-am.power2 >= mantissa_shift) { - // have a denormal float - int32_t shift = -am.power2 + 1; - cb(am, std::min(shift, 64)); - // check for round-up: if rounding-nearest carried us to the hidden bit. - am.power2 = (am.mantissa < - (uint64_t(1) << binary_format::mantissa_explicit_bits())) - ? 0 - : 1; - return; - } - - // have a normal float, use the default shift. - cb(am, mantissa_shift); - - // check for carry - if (am.mantissa >= - (uint64_t(2) << binary_format::mantissa_explicit_bits())) { - am.mantissa = (uint64_t(1) << binary_format::mantissa_explicit_bits()); - am.power2++; - } - - // check for infinite: we could have carried to an infinite power - am.mantissa &= ~(uint64_t(1) << binary_format::mantissa_explicit_bits()); - if (am.power2 >= binary_format::infinite_power()) { - am.power2 = binary_format::infinite_power(); - am.mantissa = 0; - } -} - -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void -round_nearest_tie_even(adjusted_mantissa &am, int32_t shift, - callback cb) noexcept { - const uint64_t mask = (shift == 64) ? UINT64_MAX : (uint64_t(1) << shift) - 1; - const uint64_t halfway = (shift == 0) ? 0 : uint64_t(1) << (shift - 1); - uint64_t truncated_bits = am.mantissa & mask; - bool is_above = truncated_bits > halfway; - bool is_halfway = truncated_bits == halfway; - - // shift digits into position - if (shift == 64) { - am.mantissa = 0; - } else { - am.mantissa >>= shift; - } - am.power2 += shift; - - bool is_odd = (am.mantissa & 1) == 1; - am.mantissa += uint64_t(cb(is_odd, is_halfway, is_above)); -} - -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void -round_down(adjusted_mantissa &am, int32_t shift) noexcept { - if (shift == 64) { - am.mantissa = 0; - } else { - am.mantissa >>= shift; - } - am.power2 += shift; -} -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void -skip_zeros(UC const *&first, UC const *last) noexcept { - uint64_t val; - while (!cpp20_and_in_constexpr() && - std::distance(first, last) >= int_cmp_len()) { - ::memcpy(&val, first, sizeof(uint64_t)); - if (val != int_cmp_zeros()) { - break; - } - first += int_cmp_len(); - } - while (first != last) { - if (*first != UC('0')) { - break; - } - first++; - } -} - -// determine if any non-zero digits were truncated. -// all characters must be valid digits. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool -is_truncated(UC const *first, UC const *last) noexcept { - // do 8-bit optimizations, can just compare to 8 literal 0s. - uint64_t val; - while (!cpp20_and_in_constexpr() && - std::distance(first, last) >= int_cmp_len()) { - ::memcpy(&val, first, sizeof(uint64_t)); - if (val != int_cmp_zeros()) { - return true; - } - first += int_cmp_len(); - } - while (first != last) { - if (*first != UC('0')) { - return true; - } - ++first; - } - return false; -} -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool -is_truncated(span s) noexcept { - return is_truncated(s.ptr, s.ptr + s.len()); -} - -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void -parse_eight_digits(const UC *&p, limb &value, size_t &counter, - size_t &count) noexcept { - value = value * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - counter += 8; - count += 8; -} - -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void -parse_one_digit(UC const *&p, limb &value, size_t &counter, - size_t &count) noexcept { - value = value * 10 + limb(*p - UC('0')); - p++; - counter++; - count++; -} - -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void -add_native(bigint &big, limb power, limb value) noexcept { - big.mul(power); - big.add(value); -} - -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void -round_up_bigint(bigint &big, size_t &count) noexcept { - // need to round-up the digits, but need to avoid rounding - // ....9999 to ...10000, which could cause a false halfway point. - add_native(big, 10, 1); - count++; -} - -// parse the significant digits into a big integer -template -inline FASTFLOAT_CONSTEXPR20 void -parse_mantissa(bigint &result, parsed_number_string_t &num, - size_t max_digits, size_t &digits) noexcept { - // try to minimize the number of big integer and scalar multiplication. - // therefore, try to parse 8 digits at a time, and multiply by the largest - // scalar value (9 or 19 digits) for each step. - size_t counter = 0; - digits = 0; - limb value = 0; -#ifdef FASTFLOAT_64BIT_LIMB - size_t step = 19; -#else - size_t step = 9; -#endif - - // process all integer digits. - UC const *p = num.integer.ptr; - UC const *pend = p + num.integer.len(); - skip_zeros(p, pend); - // process all digits, in increments of step per loop - while (p != pend) { - while ((std::distance(p, pend) >= 8) && (step - counter >= 8) && - (max_digits - digits >= 8)) { - parse_eight_digits(p, value, counter, digits); - } - while (counter < step && p != pend && digits < max_digits) { - parse_one_digit(p, value, counter, digits); - } - if (digits == max_digits) { - // add the temporary value, then check if we've truncated any digits - add_native(result, limb(powers_of_ten_uint64[counter]), value); - bool truncated = is_truncated(p, pend); - if (num.fraction.ptr != nullptr) { - truncated |= is_truncated(num.fraction); - } - if (truncated) { - round_up_bigint(result, digits); - } - return; - } else { - add_native(result, limb(powers_of_ten_uint64[counter]), value); - counter = 0; - value = 0; - } - } - - // add our fraction digits, if they're available. - if (num.fraction.ptr != nullptr) { - p = num.fraction.ptr; - pend = p + num.fraction.len(); - if (digits == 0) { - skip_zeros(p, pend); - } - // process all digits, in increments of step per loop - while (p != pend) { - while ((std::distance(p, pend) >= 8) && (step - counter >= 8) && - (max_digits - digits >= 8)) { - parse_eight_digits(p, value, counter, digits); - } - while (counter < step && p != pend && digits < max_digits) { - parse_one_digit(p, value, counter, digits); - } - if (digits == max_digits) { - // add the temporary value, then check if we've truncated any digits - add_native(result, limb(powers_of_ten_uint64[counter]), value); - bool truncated = is_truncated(p, pend); - if (truncated) { - round_up_bigint(result, digits); - } - return; - } else { - add_native(result, limb(powers_of_ten_uint64[counter]), value); - counter = 0; - value = 0; - } - } - } - - if (counter != 0) { - add_native(result, limb(powers_of_ten_uint64[counter]), value); - } -} - -template -inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa -positive_digit_comp(bigint &bigmant, int32_t exponent) noexcept { - FASTFLOAT_ASSERT(bigmant.pow10(uint32_t(exponent))); - adjusted_mantissa answer; - bool truncated; - answer.mantissa = bigmant.hi64(truncated); - int bias = binary_format::mantissa_explicit_bits() - - binary_format::minimum_exponent(); - answer.power2 = bigmant.bit_length() - 64 + bias; - - round(answer, [truncated](adjusted_mantissa &a, int32_t shift) { - round_nearest_tie_even( - a, shift, - [truncated](bool is_odd, bool is_halfway, bool is_above) -> bool { - return is_above || (is_halfway && truncated) || - (is_odd && is_halfway); - }); - }); - - return answer; -} - -// the scaling here is quite simple: we have, for the real digits `m * 10^e`, -// and for the theoretical digits `n * 2^f`. Since `e` is always negative, -// to scale them identically, we do `n * 2^f * 5^-f`, so we now have `m * 2^e`. -// we then need to scale by `2^(f- e)`, and then the two significant digits -// are of the same magnitude. -template -inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa negative_digit_comp( - bigint &bigmant, adjusted_mantissa am, int32_t exponent) noexcept { - bigint &real_digits = bigmant; - int32_t real_exp = exponent; - - // get the value of `b`, rounded down, and get a bigint representation of b+h - adjusted_mantissa am_b = am; - // gcc7 buf: use a lambda to remove the noexcept qualifier bug with - // -Wnoexcept-type. - round(am_b, - [](adjusted_mantissa &a, int32_t shift) { round_down(a, shift); }); - T b; - to_float(false, am_b, b); - adjusted_mantissa theor = to_extended_halfway(b); - bigint theor_digits(theor.mantissa); - int32_t theor_exp = theor.power2; - - // scale real digits and theor digits to be same power. - int32_t pow2_exp = theor_exp - real_exp; - uint32_t pow5_exp = uint32_t(-real_exp); - if (pow5_exp != 0) { - FASTFLOAT_ASSERT(theor_digits.pow5(pow5_exp)); - } - if (pow2_exp > 0) { - FASTFLOAT_ASSERT(theor_digits.pow2(uint32_t(pow2_exp))); - } else if (pow2_exp < 0) { - FASTFLOAT_ASSERT(real_digits.pow2(uint32_t(-pow2_exp))); - } - - // compare digits, and use it to director rounding - int ord = real_digits.compare(theor_digits); - adjusted_mantissa answer = am; - round(answer, [ord](adjusted_mantissa &a, int32_t shift) { - round_nearest_tie_even( - a, shift, [ord](bool is_odd, bool _, bool __) -> bool { - (void)_; // not needed, since we've done our comparison - (void)__; // not needed, since we've done our comparison - if (ord > 0) { - return true; - } else if (ord < 0) { - return false; - } else { - return is_odd; - } - }); - }); - - return answer; -} - -// parse the significant digits as a big integer to unambiguously round the -// the significant digits. here, we are trying to determine how to round -// an extended float representation close to `b+h`, halfway between `b` -// (the float rounded-down) and `b+u`, the next positive float. this -// algorithm is always correct, and uses one of two approaches. when -// the exponent is positive relative to the significant digits (such as -// 1234), we create a big-integer representation, get the high 64-bits, -// determine if any lower bits are truncated, and use that to direct -// rounding. in case of a negative exponent relative to the significant -// digits (such as 1.2345), we create a theoretical representation of -// `b` as a big-integer type, scaled to the same binary exponent as -// the actual digits. we then compare the big integer representations -// of both, and use that to direct rounding. -template -inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa -digit_comp(parsed_number_string_t &num, adjusted_mantissa am) noexcept { - // remove the invalid exponent bias - am.power2 -= invalid_am_bias; - - int32_t sci_exp = scientific_exponent(num); - size_t max_digits = binary_format::max_digits(); - size_t digits = 0; - bigint bigmant; - parse_mantissa(bigmant, num, max_digits, digits); - // can't underflow, since digits is at most max_digits. - int32_t exponent = sci_exp + 1 - int32_t(digits); - if (exponent >= 0) { - return positive_digit_comp(bigmant, exponent); - } else { - return negative_digit_comp(bigmant, am, exponent); - } -} - -} // namespace fast_float - -#endif - -#ifndef FASTFLOAT_PARSE_NUMBER_H -#define FASTFLOAT_PARSE_NUMBER_H - - -#include -#include -#include -#include -namespace fast_float { - -namespace detail { -/** - * Special case +inf, -inf, nan, infinity, -infinity. - * The case comparisons could be made much faster given that we know that the - * strings a null-free and fixed. - **/ -template -from_chars_result_t FASTFLOAT_CONSTEXPR14 parse_infnan(UC const *first, - UC const *last, - T &value) noexcept { - from_chars_result_t answer{}; - answer.ptr = first; - answer.ec = std::errc(); // be optimistic - bool minusSign = false; - if (*first == - UC('-')) { // assume first < last, so dereference without checks; - // C++17 20.19.3.(7.1) explicitly forbids '+' here - minusSign = true; - ++first; - } -#ifdef FASTFLOAT_ALLOWS_LEADING_PLUS // disabled by default - if (*first == UC('+')) { - ++first; - } -#endif - if (last - first >= 3) { - if (fastfloat_strncasecmp(first, str_const_nan(), 3)) { - answer.ptr = (first += 3); - value = minusSign ? -std::numeric_limits::quiet_NaN() - : std::numeric_limits::quiet_NaN(); - // Check for possible nan(n-char-seq-opt), C++17 20.19.3.7, - // C11 7.20.1.3.3. At least MSVC produces nan(ind) and nan(snan). - if (first != last && *first == UC('(')) { - for (UC const *ptr = first + 1; ptr != last; ++ptr) { - if (*ptr == UC(')')) { - answer.ptr = ptr + 1; // valid nan(n-char-seq-opt) - break; - } else if (!((UC('a') <= *ptr && *ptr <= UC('z')) || - (UC('A') <= *ptr && *ptr <= UC('Z')) || - (UC('0') <= *ptr && *ptr <= UC('9')) || *ptr == UC('_'))) - break; // forbidden char, not nan(n-char-seq-opt) - } - } - return answer; - } - if (fastfloat_strncasecmp(first, str_const_inf(), 3)) { - if ((last - first >= 8) && - fastfloat_strncasecmp(first + 3, str_const_inf() + 3, 5)) { - answer.ptr = first + 8; - } else { - answer.ptr = first + 3; - } - value = minusSign ? -std::numeric_limits::infinity() - : std::numeric_limits::infinity(); - return answer; - } - } - answer.ec = std::errc::invalid_argument; - return answer; -} - -/** - * Returns true if the floating-pointing rounding mode is to 'nearest'. - * It is the default on most system. This function is meant to be inexpensive. - * Credit : @mwalcott3 - */ -fastfloat_really_inline bool rounds_to_nearest() noexcept { - // https://lemire.me/blog/2020/06/26/gcc-not-nearest/ -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - return false; -#endif - // See - // A fast function to check your floating-point rounding mode - // https://lemire.me/blog/2022/11/16/a-fast-function-to-check-your-floating-point-rounding-mode/ - // - // This function is meant to be equivalent to : - // prior: #include - // return fegetround() == FE_TONEAREST; - // However, it is expected to be much faster than the fegetround() - // function call. - // - // The volatile keywoard prevents the compiler from computing the function - // at compile-time. - // There might be other ways to prevent compile-time optimizations (e.g., - // asm). The value does not need to be std::numeric_limits::min(), any - // small value so that 1 + x should round to 1 would do (after accounting for - // excess precision, as in 387 instructions). - static volatile float fmin = std::numeric_limits::min(); - float fmini = fmin; // we copy it so that it gets loaded at most once. -// -// Explanation: -// Only when fegetround() == FE_TONEAREST do we have that -// fmin + 1.0f == 1.0f - fmin. -// -// FE_UPWARD: -// fmin + 1.0f > 1 -// 1.0f - fmin == 1 -// -// FE_DOWNWARD or FE_TOWARDZERO: -// fmin + 1.0f == 1 -// 1.0f - fmin < 1 -// -// Note: This may fail to be accurate if fast-math has been -// enabled, as rounding conventions may not apply. -#ifdef FASTFLOAT_VISUAL_STUDIO -#pragma warning(push) -// todo: is there a VS warning? -// see -// https://stackoverflow.com/questions/46079446/is-there-a-warning-for-floating-point-equality-checking-in-visual-studio-2013 -#elif defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wfloat-equal" -#elif defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" -#endif - return (fmini + 1.0f == 1.0f - fmini); -#ifdef FASTFLOAT_VISUAL_STUDIO -#pragma warning(pop) -#elif defined(__clang__) -#pragma clang diagnostic pop -#elif defined(__GNUC__) -#pragma GCC diagnostic pop -#endif -} - -} // namespace detail - -template struct from_chars_caller { - template - FASTFLOAT_CONSTEXPR20 static from_chars_result_t - call(UC const *first, UC const *last, T &value, - parse_options_t options) noexcept { - return from_chars_advanced(first, last, value, options); - } -}; - -#if __STDCPP_FLOAT32_T__ == 1 -template <> struct from_chars_caller { - template - FASTFLOAT_CONSTEXPR20 static from_chars_result_t - call(UC const *first, UC const *last, std::float32_t &value, - parse_options_t options) noexcept { - // if std::float32_t is defined, and we are in C++23 mode; macro set for - // float32; set value to float due to equivalence between float and - // float32_t - float val; - auto ret = from_chars_advanced(first, last, val, options); - value = val; - return ret; - } -}; -#endif - -#if __STDCPP_FLOAT64_T__ == 1 -template <> struct from_chars_caller { - template - FASTFLOAT_CONSTEXPR20 static from_chars_result_t - call(UC const *first, UC const *last, std::float64_t &value, - parse_options_t options) noexcept { - // if std::float64_t is defined, and we are in C++23 mode; macro set for - // float64; set value as double due to equivalence between double and - // float64_t - double val; - auto ret = from_chars_advanced(first, last, val, options); - value = val; - return ret; - } -}; -#endif - -template -FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars(UC const *first, UC const *last, T &value, - chars_format fmt /*= chars_format::general*/) noexcept { - return from_chars_caller::call(first, last, value, - parse_options_t(fmt)); -} - -/** - * This function overload takes parsed_number_string_t structure that is created - * and populated either by from_chars_advanced function taking chars range and - * parsing options or other parsing custom function implemented by user. - */ -template -FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars_advanced(parsed_number_string_t &pns, T &value) noexcept { - - static_assert(is_supported_float_type(), - "only some floating-point types are supported"); - static_assert(is_supported_char_type(), - "only char, wchar_t, char16_t and char32_t are supported"); - - from_chars_result_t answer; - - answer.ec = std::errc(); // be optimistic - answer.ptr = pns.lastmatch; - // The implementation of the Clinger's fast path is convoluted because - // we want round-to-nearest in all cases, irrespective of the rounding mode - // selected on the thread. - // We proceed optimistically, assuming that detail::rounds_to_nearest() - // returns true. - if (binary_format::min_exponent_fast_path() <= pns.exponent && - pns.exponent <= binary_format::max_exponent_fast_path() && - !pns.too_many_digits) { - // Unfortunately, the conventional Clinger's fast path is only possible - // when the system rounds to the nearest float. - // - // We expect the next branch to almost always be selected. - // We could check it first (before the previous branch), but - // there might be performance advantages at having the check - // be last. - if (!cpp20_and_in_constexpr() && detail::rounds_to_nearest()) { - // We have that fegetround() == FE_TONEAREST. - // Next is Clinger's fast path. - if (pns.mantissa <= binary_format::max_mantissa_fast_path()) { - value = T(pns.mantissa); - if (pns.exponent < 0) { - value = value / binary_format::exact_power_of_ten(-pns.exponent); - } else { - value = value * binary_format::exact_power_of_ten(pns.exponent); - } - if (pns.negative) { - value = -value; - } - return answer; - } - } else { - // We do not have that fegetround() == FE_TONEAREST. - // Next is a modified Clinger's fast path, inspired by Jakub Jelínek's - // proposal - if (pns.exponent >= 0 && - pns.mantissa <= - binary_format::max_mantissa_fast_path(pns.exponent)) { -#if defined(__clang__) || defined(FASTFLOAT_32BIT) - // Clang may map 0 to -0.0 when fegetround() == FE_DOWNWARD - if (pns.mantissa == 0) { - value = pns.negative ? T(-0.) : T(0.); - return answer; - } -#endif - value = T(pns.mantissa) * - binary_format::exact_power_of_ten(pns.exponent); - if (pns.negative) { - value = -value; - } - return answer; - } - } - } - adjusted_mantissa am = - compute_float>(pns.exponent, pns.mantissa); - if (pns.too_many_digits && am.power2 >= 0) { - if (am != compute_float>(pns.exponent, pns.mantissa + 1)) { - am = compute_error>(pns.exponent, pns.mantissa); - } - } - // If we called compute_float>(pns.exponent, pns.mantissa) - // and we have an invalid power (am.power2 < 0), then we need to go the long - // way around again. This is very uncommon. - if (am.power2 < 0) { - am = digit_comp(pns, am); - } - to_float(pns.negative, am, value); - // Test for over/underflow. - if ((pns.mantissa != 0 && am.mantissa == 0 && am.power2 == 0) || - am.power2 == binary_format::infinite_power()) { - answer.ec = std::errc::result_out_of_range; - } - return answer; -} - -template -FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars_advanced(UC const *first, UC const *last, T &value, - parse_options_t options) noexcept { - - static_assert(is_supported_float_type(), - "only some floating-point types are supported"); - static_assert(is_supported_char_type(), - "only char, wchar_t, char16_t and char32_t are supported"); - - from_chars_result_t answer; -#ifdef FASTFLOAT_SKIP_WHITE_SPACE // disabled by default - while ((first != last) && fast_float::is_space(uint8_t(*first))) { - first++; - } -#endif - if (first == last) { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - return answer; - } - parsed_number_string_t pns = - parse_number_string(first, last, options); - if (!pns.valid) { - if (options.format & chars_format::no_infnan) { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - return answer; - } else { - return detail::parse_infnan(first, last, value); - } - } - - // call overload that takes parsed_number_string_t directly. - return from_chars_advanced(pns, value); -} - -template -FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars(UC const *first, UC const *last, T &value, int base) noexcept { - static_assert(is_supported_char_type(), - "only char, wchar_t, char16_t and char32_t are supported"); - - from_chars_result_t answer; -#ifdef FASTFLOAT_SKIP_WHITE_SPACE // disabled by default - while ((first != last) && fast_float::is_space(uint8_t(*first))) { - first++; - } -#endif - if (first == last || base < 2 || base > 36) { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - return answer; - } - return parse_int_string(first, last, value, base); -} - -} // namespace fast_float - -#endif - diff --git a/deps/fast_float/fast_float_strtod.cpp b/deps/fast_float/fast_float_strtod.cpp deleted file mode 100644 index 7f4235c7e..000000000 --- a/deps/fast_float/fast_float_strtod.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include "fast_float.h" -#include -#include -#include -#include - -/* Convert NPTR to a double using the fast_float library. - * - * This function behaves similarly to the standard strtod function, converting - * the initial portion of the string pointed to by `nptr` to a `double` value, - * using the fast_float library for high performance. If the conversion fails, - * errno is set to EINVAL error code. - * - * @param nptr A pointer to the null-terminated byte string to be interpreted. - * @param endptr A pointer to a pointer to character. If `endptr` is not NULL, - * it will point to the character after the last character used - * in the conversion. - * @return The converted value as a double. If no valid conversion could - * be performed, returns 0.0. - * If ENDPTR is not NULL, a pointer to the character after the last one used - * in the number is put in *ENDPTR. */ -extern "C" double fast_float_strtod(const char *nptr, char **endptr) { - double result = 0.0; - auto answer = fast_float::from_chars(nptr, nptr + strlen(nptr), result); - if (answer.ec != std::errc()) { - errno = EINVAL; // Fallback to for other errors - } - if (endptr != NULL) { - *endptr = (char *)answer.ptr; - } - return result; -} diff --git a/deps/fast_float/fast_float_strtod.h b/deps/fast_float/fast_float_strtod.h deleted file mode 100644 index 1755076a1..000000000 --- a/deps/fast_float/fast_float_strtod.h +++ /dev/null @@ -1,15 +0,0 @@ - -#ifndef __FAST_FLOAT_STRTOD_H__ -#define __FAST_FLOAT_STRTOD_H__ - -#if defined(__cplusplus) -extern "C" -{ -#endif - double fast_float_strtod(const char *in, char **out); - -#if defined(__cplusplus) -} -#endif - -#endif /* __FAST_FLOAT_STRTOD_H__ */ diff --git a/modules/redisbloom/Makefile b/modules/redisbloom/Makefile index 1e113b400..f40cc7c1f 100644 --- a/modules/redisbloom/Makefile +++ b/modules/redisbloom/Makefile @@ -1,5 +1,5 @@ SRC_DIR = src -MODULE_VERSION = v8.7.80 +MODULE_VERSION = v8.7.90 MODULE_REPO = https://github.com/redisbloom/redisbloom TARGET_MODULE = $(SRC_DIR)/bin/$(FULL_VARIANT)/redisbloom.so diff --git a/modules/redisjson/Makefile b/modules/redisjson/Makefile index 46ee46b88..4d13ed7bc 100644 --- a/modules/redisjson/Makefile +++ b/modules/redisjson/Makefile @@ -1,5 +1,5 @@ SRC_DIR = src -MODULE_VERSION = v8.7.80 +MODULE_VERSION = v8.7.90 MODULE_REPO = https://github.com/redisjson/redisjson TARGET_MODULE = $(SRC_DIR)/bin/$(FULL_VARIANT)/rejson.so diff --git a/modules/redistimeseries/Makefile b/modules/redistimeseries/Makefile index 030e73aaf..1bd8b46ca 100644 --- a/modules/redistimeseries/Makefile +++ b/modules/redistimeseries/Makefile @@ -1,5 +1,5 @@ SRC_DIR = src -MODULE_VERSION = v8.7.80 +MODULE_VERSION = v8.7.90 MODULE_REPO = https://github.com/redistimeseries/redistimeseries TARGET_MODULE = $(SRC_DIR)/bin/$(FULL_VARIANT)/redistimeseries.so diff --git a/redis.conf b/redis.conf index 30a3d8b57..845be292f 100644 --- a/redis.conf +++ b/redis.conf @@ -2039,8 +2039,9 @@ latency-monitor-threshold 0 # (Note: not included in the 'A' class) # c Type-changed events generated every time a key's type changes # (Note: not included in the 'A' class) +# r rate limit event # A Alias for g$lshzxetd, so that the "AKE" string means all the events -# except key-miss, new key, overwritten and type-changed. +# except key-miss, new key, overwritten, type-changed and rate-limit. # # The "notify-keyspace-events" takes as argument a string that is composed # of zero or multiple characters. The empty string means that notifications diff --git a/src/Makefile b/src/Makefile index 36e4b5b4f..046aaa5aa 100644 --- a/src/Makefile +++ b/src/Makefile @@ -35,7 +35,7 @@ endif ifneq ($(OPTIMIZATION),-O0) OPTIMIZATION+=-fno-omit-frame-pointer endif -DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram fpconv fast_float xxhash +DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram fpconv xxhash NODEPS:=clean distclean # Default settings @@ -149,7 +149,7 @@ endif FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) FINAL_LDFLAGS=$(LDFLAGS) $(OPT) $(REDIS_LDFLAGS) $(DEBUG) -FINAL_LIBS=-lm -lstdc++ +FINAL_LIBS=-lm DEBUG=-g -ggdb # Linux ARM32 needs -latomic at linking time @@ -257,7 +257,7 @@ ifdef OPENSSL_PREFIX endif # Include paths to dependencies -FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/fpconv -I../deps/fast_float -I../deps/xxhash +FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/fpconv -I../deps/xxhash # Determine systemd support and/or build preference (defaulting to auto-detection) BUILD_WITH_SYSTEMD=no @@ -382,7 +382,7 @@ endif REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX) REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX) -REDIS_SERVER_OBJ=threads_mngr.o memory_prefetch.o adlist.o quicklist.o ae.o anet.o dict.o ebuckets.o eventnotifier.o iothread.o mstr.o entry.o kvstore.o fwtree.o estore.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o cluster_asm.o cluster_legacy.o cluster_slot_stats.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crccombine.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o flax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o lolwut8.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o functions.o function_lua.o commands.o strl.o connection.o unix.o logreqres.o keymeta.o chk.o hotkeys.o gcra.o +REDIS_SERVER_OBJ=threads_mngr.o memory_prefetch.o adlist.o quicklist.o ae.o anet.o dict.o ebuckets.o eventnotifier.o iothread.o mstr.o entry.o kvstore.o fwtree.o estore.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o cluster_asm.o cluster_legacy.o cluster_slot_stats.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crccombine.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o flax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o lolwut8.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o functions.o function_lua.o commands.o strl.o connection.o unix.o logreqres.o keymeta.o chk.o hotkeys.o gcra.o vector.o fast_float_strtod.o REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX) REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o redisassert.o crcspeed.o crccombine.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o strl.o cli_commands.o REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX) @@ -442,7 +442,7 @@ endif # redis-server $(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) $(REDIS_VEC_SETS_OBJ) - $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a ../deps/fast_float/libfast_float.a ../deps/xxhash/libxxhash.a $(FINAL_LIBS) + $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a ../deps/xxhash/libxxhash.a $(FINAL_LIBS) # redis-sentinel $(REDIS_SENTINEL_NAME): $(REDIS_SERVER_NAME) diff --git a/src/acl.c b/src/acl.c index e7c61aacf..79a900200 100644 --- a/src/acl.c +++ b/src/acl.c @@ -70,6 +70,7 @@ struct ACLCategoryItem { {"connection", ACL_CATEGORY_CONNECTION}, {"transaction", ACL_CATEGORY_TRANSACTION}, {"scripting", ACL_CATEGORY_SCRIPTING}, + {"ratelimit", ACL_CATEGORY_RATE_LIMIT}, {NULL,0} /* Terminator. */ }; diff --git a/src/aof.c b/src/aof.c index b8e50584d..d58996877 100644 --- a/src/aof.c +++ b/src/aof.c @@ -2467,6 +2467,18 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { return 1; } +int rewriteGCRAObject(rio *r, robj *key, robj *o) { + long long val; + getLongLongFromGCRAObject(o, &val); + + /* GCRASETVALUE */ + if (rioWriteBulkCount(r,'*',3) == 0) return 0; + if (rioWriteBulkString(r,"GCRASETVALUE",12) == 0) return 0; + if (rioWriteBulkObject(r,key) == 0) return 0; + if (rioWriteBulkLongLong(r,val) == 0) return 0; + return 1; +} + /* Call the module type callback in order to rewrite a data type * that is exported by a module and is not handled by Redis itself. * The function returns 0 on error, 1 on success. */ @@ -2522,6 +2534,8 @@ int rewriteObject(rio *r, robj *key, robj *o, int dbid, long long expiretime) { if (rewriteHashObject(r,key,o) == 0) return C_ERR; } else if (o->type == OBJ_STREAM) { if (rewriteStreamObject(r,key,o) == 0) return C_ERR; + } else if (o->type == OBJ_GCRA) { + if (rewriteGCRAObject(r,key,o) == 0) return C_ERR; } else if (o->type == OBJ_MODULE) { if (rewriteModuleObject(r,key,o,dbid) == 0) return C_ERR; } else { @@ -2570,10 +2584,20 @@ int rewriteAppendOnlyFileRio(rio *aof) { if (rioWriteBulkLongLong(aof,j) == 0) goto werr; kvstoreIteratorInit(&kvs_it, db->keys); + int last_slot = -1; /* Iterate this DB writing every entry */ while((de = kvstoreIteratorNext(&kvs_it)) != NULL) { long long expiretime; size_t aof_bytes_before_key = aof->processed_bytes; + int curr_slot = kvstoreIteratorGetCurrentDictIndex(&kvs_it); + + /* In cluster mode, dismiss bucket arrays of the previous slot + * which won't be accessed again, to avoid CoW. */ + if (server.cluster_enabled && curr_slot != last_slot) { + if (server.in_fork_child && last_slot != -1) + dismissDictBucketsMemory(kvstoreGetDict(db->keys, last_slot)); + last_slot = curr_slot; + } /* Get the value object (of type kvobj) */ kvobj *o = dictGetKV(de); @@ -2582,12 +2606,9 @@ int rewriteAppendOnlyFileRio(rio *aof) { expiretime = kvobjGetExpire(o); /* Skip keys that are being trimmed */ - if (server.cluster_enabled) { - int curr_slot = kvstoreIteratorGetCurrentDictIndex(&kvs_it); - if (isSlotInTrimJob(curr_slot)) { - skipped++; - continue; - } + if (server.cluster_enabled && isSlotInTrimJob(curr_slot)) { + skipped++; + continue; } /* Set on stack string object for key */ @@ -2600,7 +2621,8 @@ int rewriteAppendOnlyFileRio(rio *aof) { * OS and possibly avoid or decrease COW. We give the dismiss * mechanism a hint about an estimated size of the object we stored. */ size_t dump_size = aof->processed_bytes - aof_bytes_before_key; - if (server.in_fork_child) dismissObject(o, dump_size); + if (server.in_fork_child && dump_size > server.page_size/2) + dismissObject(o, dump_size); /* Update info every 1 second (approximately). * in order to avoid calling mstime() on each iteration, we will @@ -2618,6 +2640,10 @@ int rewriteAppendOnlyFileRio(rio *aof) { debugDelay(server.rdb_key_save_delay); } kvstoreIteratorReset(&kvs_it); + + /* Dismiss bucket arrays of kvstore in standalone mode. */ + if (server.in_fork_child && !server.cluster_enabled) + dismissKvstoreBucketsMemory(db->keys); } serverLog(LL_NOTICE, "AOF rewrite done, %ld keys saved, %llu keys skipped.", key_count, skipped); return C_OK; diff --git a/src/cluster_asm.c b/src/cluster_asm.c index db801a9a1..01a071167 100644 --- a/src/cluster_asm.c +++ b/src/cluster_asm.c @@ -1057,13 +1057,27 @@ void clusterMigrationCommand(client *c) { } } +/* Returns the address of the node in the format "ip:port". */ +static const char *getNodeAddressStr(const char *node_id, int len) { + serverAssert(node_id != NULL); + static char buf[NET_HOST_PORT_STR_LEN]; + + clusterNode *n = clusterLookupNode(node_id, len); + char *ip = n ? clusterNodeIp(n) : "?"; + int port = n ? (server.tls_replication ? clusterNodeTlsPort(n) : + clusterNodeTcpPort(n)) : 0; + formatAddr(buf, sizeof(buf), ip, port); + return buf; +} + /* Log a human-readable message for ASM task lifecycle events. */ void asmLogTaskEvent(asmTask *task, int event) { sds str = slotRangeArrayToString(task->slots); switch (event) { case ASM_EVENT_IMPORT_STARTED: - serverLog(LL_NOTICE, "Import task %s started for slots: %s", task->id, str); + serverLog(LL_NOTICE, "Import task %s started for slots: %s, source address: %s", + task->id, str, getNodeAddressStr(task->source, CLUSTER_NAMELEN)); break; case ASM_EVENT_IMPORT_FAILED: serverLog(LL_NOTICE, "Import task %s failed for slots: %s", task->id, str); @@ -1076,8 +1090,8 @@ void asmLogTaskEvent(asmTask *task, int event) { task->id, str, getKeyCountInSlotRangeArray(task->slots)); break; case ASM_EVENT_MIGRATE_STARTED: - serverLog(LL_NOTICE, "Migrate task %s started for slots: %s (number of keys at start: %llu)", - task->id, str, getKeyCountInSlotRangeArray(task->slots)); + serverLog(LL_NOTICE, "Migrate task %s started for slots: %s, destination address: %s, (number of keys at start: %llu)", + task->id, str, getNodeAddressStr(task->dest, CLUSTER_NAMELEN), getKeyCountInSlotRangeArray(task->slots)); break; case ASM_EVENT_MIGRATE_FAILED: serverLog(LL_NOTICE, "Migrate task %s failed for slots: %s", task->id, str); @@ -3033,7 +3047,7 @@ void asmTriggerBackgroundTrim(asmTrimCtx *trim_ctx, int migration_cleanup) { CLUSTER_SLOT_MASK_BITS, KVSTORE_ALLOCATE_DICTS_ON_DEMAND); estore *subexpires = estoreCreate(&subexpiresBucketsType, CLUSTER_SLOT_MASK_BITS); - dict *stream_idmp_keys = dictCreate(&objectKeyPointerValueDictType); + dict *stream_idmp_keys = dictCreate(&objectKeyNoValueDictType); size_t total_keys = 0; diff --git a/src/commands.def b/src/commands.def index 07e1dccc6..fed08eda3 100644 --- a/src/commands.def +++ b/src/commands.def @@ -24,7 +24,8 @@ const char *COMMAND_GROUP_STR[] = { "geo", "stream", "bitmap", - "module" + "module", + "rate_limit" }; const char *commandGroupStr(int index) { @@ -5379,6 +5380,59 @@ struct COMMAND_ARG UNSUBSCRIBE_Args[] = { {MAKE_ARG("channel",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL|CMD_ARG_MULTIPLE,0,NULL)}, }; +/********** GCRA ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* GCRA history */ +#define GCRA_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* GCRA tips */ +#define GCRA_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* GCRA key specs */ +keySpec GCRA_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* GCRA argument table */ +struct COMMAND_ARG GCRA_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("max-burst",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("tokens-per-period",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("period",ARG_TYPE_DOUBLE,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("count",ARG_TYPE_INTEGER,-1,"TOKENS",NULL,NULL,CMD_ARG_OPTIONAL,0,NULL)}, +}; + +/********** GCRASETVALUE ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* GCRASETVALUE history */ +#define GCRASETVALUE_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* GCRASETVALUE tips */ +#define GCRASETVALUE_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* GCRASETVALUE key specs */ +keySpec GCRASETVALUE_Keyspecs[1] = { +{NULL,CMD_KEY_OW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* GCRASETVALUE argument table */ +struct COMMAND_ARG GCRASETVALUE_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("tat",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + /********** EVAL ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -9165,7 +9219,9 @@ struct COMMAND_ARG ZINCRBY_Args[] = { #ifndef SKIP_CMD_HISTORY_TABLE /* ZINTER history */ -#define ZINTER_History NULL +commandHistory ZINTER_History[] = { +{"8.8.0","Added `COUNT` aggregate option."}, +}; #endif #ifndef SKIP_CMD_TIPS_TABLE @@ -9185,6 +9241,7 @@ struct COMMAND_ARG ZINTER_aggregate_Subargs[] = { {MAKE_ARG("sum",ARG_TYPE_PURE_TOKEN,-1,"SUM",NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("min",ARG_TYPE_PURE_TOKEN,-1,"MIN",NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("max",ARG_TYPE_PURE_TOKEN,-1,"MAX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("count",ARG_TYPE_PURE_TOKEN,-1,"COUNT",NULL,"8.8.0",CMD_ARG_NONE,0,NULL)}, }; /* ZINTER argument table */ @@ -9192,7 +9249,7 @@ struct COMMAND_ARG ZINTER_Args[] = { {MAKE_ARG("numkeys",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, {MAKE_ARG("weight",ARG_TYPE_INTEGER,-1,"WEIGHTS",NULL,NULL,CMD_ARG_OPTIONAL|CMD_ARG_MULTIPLE,0,NULL)}, -{MAKE_ARG("aggregate",ARG_TYPE_ONEOF,-1,"AGGREGATE",NULL,NULL,CMD_ARG_OPTIONAL,3,NULL),.subargs=ZINTER_aggregate_Subargs}, +{MAKE_ARG("aggregate",ARG_TYPE_ONEOF,-1,"AGGREGATE",NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=ZINTER_aggregate_Subargs}, {MAKE_ARG("withscores",ARG_TYPE_PURE_TOKEN,-1,"WITHSCORES",NULL,NULL,CMD_ARG_OPTIONAL,0,NULL)}, }; @@ -9226,7 +9283,9 @@ struct COMMAND_ARG ZINTERCARD_Args[] = { #ifndef SKIP_CMD_HISTORY_TABLE /* ZINTERSTORE history */ -#define ZINTERSTORE_History NULL +commandHistory ZINTERSTORE_History[] = { +{"8.8.0","Added `COUNT` aggregate option."}, +}; #endif #ifndef SKIP_CMD_TIPS_TABLE @@ -9246,6 +9305,7 @@ struct COMMAND_ARG ZINTERSTORE_aggregate_Subargs[] = { {MAKE_ARG("sum",ARG_TYPE_PURE_TOKEN,-1,"SUM",NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("min",ARG_TYPE_PURE_TOKEN,-1,"MIN",NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("max",ARG_TYPE_PURE_TOKEN,-1,"MAX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("count",ARG_TYPE_PURE_TOKEN,-1,"COUNT",NULL,"8.8.0",CMD_ARG_NONE,0,NULL)}, }; /* ZINTERSTORE argument table */ @@ -9254,7 +9314,7 @@ struct COMMAND_ARG ZINTERSTORE_Args[] = { {MAKE_ARG("numkeys",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("key",ARG_TYPE_KEY,1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, {MAKE_ARG("weight",ARG_TYPE_INTEGER,-1,"WEIGHTS",NULL,NULL,CMD_ARG_OPTIONAL|CMD_ARG_MULTIPLE,0,NULL)}, -{MAKE_ARG("aggregate",ARG_TYPE_ONEOF,-1,"AGGREGATE",NULL,NULL,CMD_ARG_OPTIONAL,3,NULL),.subargs=ZINTERSTORE_aggregate_Subargs}, +{MAKE_ARG("aggregate",ARG_TYPE_ONEOF,-1,"AGGREGATE",NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=ZINTERSTORE_aggregate_Subargs}, }; /********** ZLEXCOUNT ********************/ @@ -9894,7 +9954,9 @@ struct COMMAND_ARG ZSCORE_Args[] = { #ifndef SKIP_CMD_HISTORY_TABLE /* ZUNION history */ -#define ZUNION_History NULL +commandHistory ZUNION_History[] = { +{"8.8.0","Added `COUNT` aggregate option."}, +}; #endif #ifndef SKIP_CMD_TIPS_TABLE @@ -9914,6 +9976,7 @@ struct COMMAND_ARG ZUNION_aggregate_Subargs[] = { {MAKE_ARG("sum",ARG_TYPE_PURE_TOKEN,-1,"SUM",NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("min",ARG_TYPE_PURE_TOKEN,-1,"MIN",NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("max",ARG_TYPE_PURE_TOKEN,-1,"MAX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("count",ARG_TYPE_PURE_TOKEN,-1,"COUNT",NULL,"8.8.0",CMD_ARG_NONE,0,NULL)}, }; /* ZUNION argument table */ @@ -9921,7 +9984,7 @@ struct COMMAND_ARG ZUNION_Args[] = { {MAKE_ARG("numkeys",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, {MAKE_ARG("weight",ARG_TYPE_INTEGER,-1,"WEIGHTS",NULL,NULL,CMD_ARG_OPTIONAL|CMD_ARG_MULTIPLE,0,NULL)}, -{MAKE_ARG("aggregate",ARG_TYPE_ONEOF,-1,"AGGREGATE",NULL,NULL,CMD_ARG_OPTIONAL,3,NULL),.subargs=ZUNION_aggregate_Subargs}, +{MAKE_ARG("aggregate",ARG_TYPE_ONEOF,-1,"AGGREGATE",NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=ZUNION_aggregate_Subargs}, {MAKE_ARG("withscores",ARG_TYPE_PURE_TOKEN,-1,"WITHSCORES",NULL,NULL,CMD_ARG_OPTIONAL,0,NULL)}, }; @@ -9929,7 +9992,9 @@ struct COMMAND_ARG ZUNION_Args[] = { #ifndef SKIP_CMD_HISTORY_TABLE /* ZUNIONSTORE history */ -#define ZUNIONSTORE_History NULL +commandHistory ZUNIONSTORE_History[] = { +{"8.8.0","Added `COUNT` aggregate option."}, +}; #endif #ifndef SKIP_CMD_TIPS_TABLE @@ -9949,6 +10014,7 @@ struct COMMAND_ARG ZUNIONSTORE_aggregate_Subargs[] = { {MAKE_ARG("sum",ARG_TYPE_PURE_TOKEN,-1,"SUM",NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("min",ARG_TYPE_PURE_TOKEN,-1,"MIN",NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("max",ARG_TYPE_PURE_TOKEN,-1,"MAX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("count",ARG_TYPE_PURE_TOKEN,-1,"COUNT",NULL,"8.8.0",CMD_ARG_NONE,0,NULL)}, }; /* ZUNIONSTORE argument table */ @@ -9957,7 +10023,7 @@ struct COMMAND_ARG ZUNIONSTORE_Args[] = { {MAKE_ARG("numkeys",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("key",ARG_TYPE_KEY,1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, {MAKE_ARG("weight",ARG_TYPE_INTEGER,-1,"WEIGHTS",NULL,NULL,CMD_ARG_OPTIONAL|CMD_ARG_MULTIPLE,0,NULL)}, -{MAKE_ARG("aggregate",ARG_TYPE_ONEOF,-1,"AGGREGATE",NULL,NULL,CMD_ARG_OPTIONAL,3,NULL),.subargs=ZUNIONSTORE_aggregate_Subargs}, +{MAKE_ARG("aggregate",ARG_TYPE_ONEOF,-1,"AGGREGATE",NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=ZUNIONSTORE_aggregate_Subargs}, }; /********** XACK ********************/ @@ -11082,34 +11148,6 @@ struct COMMAND_ARG DIGEST_Args[] = { {MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, }; -/********** GCRA ********************/ - -#ifndef SKIP_CMD_HISTORY_TABLE -/* GCRA history */ -#define GCRA_History NULL -#endif - -#ifndef SKIP_CMD_TIPS_TABLE -/* GCRA tips */ -#define GCRA_Tips NULL -#endif - -#ifndef SKIP_CMD_KEY_SPECS_TABLE -/* GCRA key specs */ -keySpec GCRA_Keyspecs[1] = { -{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} -}; -#endif - -/* GCRA argument table */ -struct COMMAND_ARG GCRA_Args[] = { -{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, -{MAKE_ARG("max-burst",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, -{MAKE_ARG("tokens-per-period",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, -{MAKE_ARG("period",ARG_TYPE_DOUBLE,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, -{MAKE_ARG("count",ARG_TYPE_INTEGER,-1,"TOKENS",NULL,NULL,CMD_ARG_OPTIONAL,0,NULL)}, -}; - /********** GET ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -11917,6 +11955,9 @@ struct COMMAND_STRUCT redisCommandTable[] = { {MAKE_CMD("subscribe","Listens for messages published to channels.","O(N) where N is the number of channels to subscribe to.","2.0.0",CMD_DOC_NONE,NULL,NULL,"pubsub",COMMAND_GROUP_PUBSUB,SUBSCRIBE_History,0,SUBSCRIBE_Tips,0,subscribeCommand,-2,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,0,SUBSCRIBE_Keyspecs,0,NULL,1),.args=SUBSCRIBE_Args}, {MAKE_CMD("sunsubscribe","Stops listening to messages posted to shard channels.","O(N) where N is the number of shard channels to unsubscribe.","7.0.0",CMD_DOC_NONE,NULL,NULL,"pubsub",COMMAND_GROUP_PUBSUB,SUNSUBSCRIBE_History,0,SUNSUBSCRIBE_Tips,0,sunsubscribeCommand,-1,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0,SUNSUBSCRIBE_Keyspecs,1,NULL,1),.args=SUNSUBSCRIBE_Args}, {MAKE_CMD("unsubscribe","Stops listening to messages posted to channels.","O(N) where N is the number of channels to unsubscribe.","2.0.0",CMD_DOC_NONE,NULL,NULL,"pubsub",COMMAND_GROUP_PUBSUB,UNSUBSCRIBE_History,0,UNSUBSCRIBE_Tips,0,unsubscribeCommand,-1,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,0,UNSUBSCRIBE_Keyspecs,0,NULL,1),.args=UNSUBSCRIBE_Args}, +/* rate_limit */ +{MAKE_CMD("gcra","Rate limit via GCRA (Generic Cell Rate Algorithm).","O(1)","8.8.0",CMD_DOC_NONE,NULL,NULL,"rate_limit",COMMAND_GROUP_RATE_LIMIT,GCRA_History,0,GCRA_Tips,0,gcraCommand,-5,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_RATE_LIMIT,GCRA_Keyspecs,1,NULL,5),.args=GCRA_Args}, +{MAKE_CMD("gcrasetvalue","An internal command for recording a GCRA TAT value during AOF rewrite and replication.","O(1)","8.8.0",CMD_DOC_NONE,NULL,NULL,"rate_limit",COMMAND_GROUP_RATE_LIMIT,GCRASETVALUE_History,0,GCRASETVALUE_Tips,0,gcraSetValueCommand,3,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_RATE_LIMIT,GCRASETVALUE_Keyspecs,1,NULL,2),.args=GCRASETVALUE_Args}, /* scripting */ {MAKE_CMD("eval","Executes a server-side Lua script.","Depends on the script that is executed.","2.6.0",CMD_DOC_NONE,NULL,NULL,"scripting",COMMAND_GROUP_SCRIPTING,EVAL_History,0,EVAL_Tips,0,evalCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,EVAL_Keyspecs,1,evalGetKeys,4),.args=EVAL_Args}, {MAKE_CMD("evalsha","Executes a server-side Lua script by SHA1 digest.","Depends on the script that is executed.","2.6.0",CMD_DOC_NONE,NULL,NULL,"scripting",COMMAND_GROUP_SCRIPTING,EVALSHA_History,0,EVALSHA_Tips,0,evalShaCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,EVALSHA_Keyspecs,1,evalGetKeys,4),.args=EVALSHA_Args}, @@ -11988,9 +12029,9 @@ struct COMMAND_STRUCT redisCommandTable[] = { {MAKE_CMD("zdiff","Returns the difference between multiple sorted sets.","O(L + (N-K)log(N)) worst case where L is the total number of elements in all the sets, N is the size of the first set, and K is the size of the result set.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZDIFF_History,0,ZDIFF_Tips,0,zdiffCommand,-3,CMD_READONLY,ACL_CATEGORY_SORTEDSET,ZDIFF_Keyspecs,1,zunionInterDiffGetKeys,3),.args=ZDIFF_Args}, {MAKE_CMD("zdiffstore","Stores the difference of multiple sorted sets in a key.","O(L + (N-K)log(N)) worst case where L is the total number of elements in all the sets, N is the size of the first set, and K is the size of the result set.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZDIFFSTORE_History,0,ZDIFFSTORE_Tips,0,zdiffstoreCommand,-4,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SORTEDSET,ZDIFFSTORE_Keyspecs,2,zunionInterDiffStoreGetKeys,3),.args=ZDIFFSTORE_Args}, {MAKE_CMD("zincrby","Increments the score of a member in a sorted set.","O(log(N)) where N is the number of elements in the sorted set.","1.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZINCRBY_History,0,ZINCRBY_Tips,0,zincrbyCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_SORTEDSET,ZINCRBY_Keyspecs,1,NULL,3),.args=ZINCRBY_Args}, -{MAKE_CMD("zinter","Returns the intersect of multiple sorted sets.","O(N*K)+O(M*log(M)) worst case with N being the smallest input sorted set, K being the number of input sorted sets and M being the number of elements in the resulting sorted set.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZINTER_History,0,ZINTER_Tips,0,zinterCommand,-3,CMD_READONLY,ACL_CATEGORY_SORTEDSET,ZINTER_Keyspecs,1,zunionInterDiffGetKeys,5),.args=ZINTER_Args}, +{MAKE_CMD("zinter","Returns the intersect of multiple sorted sets.","O(N*K)+O(M*log(M)) worst case with N being the smallest input sorted set, K being the number of input sorted sets and M being the number of elements in the resulting sorted set.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZINTER_History,1,ZINTER_Tips,0,zinterCommand,-3,CMD_READONLY,ACL_CATEGORY_SORTEDSET,ZINTER_Keyspecs,1,zunionInterDiffGetKeys,5),.args=ZINTER_Args}, {MAKE_CMD("zintercard","Returns the number of members of the intersect of multiple sorted sets.","O(N*K) worst case with N being the smallest input sorted set, K being the number of input sorted sets.","7.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZINTERCARD_History,0,ZINTERCARD_Tips,0,zinterCardCommand,-3,CMD_READONLY,ACL_CATEGORY_SORTEDSET,ZINTERCARD_Keyspecs,1,zunionInterDiffGetKeys,3),.args=ZINTERCARD_Args}, -{MAKE_CMD("zinterstore","Stores the intersect of multiple sorted sets in a key.","O(N*K)+O(M*log(M)) worst case with N being the smallest input sorted set, K being the number of input sorted sets and M being the number of elements in the resulting sorted set.","2.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZINTERSTORE_History,0,ZINTERSTORE_Tips,0,zinterstoreCommand,-4,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SORTEDSET,ZINTERSTORE_Keyspecs,2,zunionInterDiffStoreGetKeys,5),.args=ZINTERSTORE_Args}, +{MAKE_CMD("zinterstore","Stores the intersect of multiple sorted sets in a key.","O(N*K)+O(M*log(M)) worst case with N being the smallest input sorted set, K being the number of input sorted sets and M being the number of elements in the resulting sorted set.","2.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZINTERSTORE_History,1,ZINTERSTORE_Tips,0,zinterstoreCommand,-4,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SORTEDSET,ZINTERSTORE_Keyspecs,2,zunionInterDiffStoreGetKeys,5),.args=ZINTERSTORE_Args}, {MAKE_CMD("zlexcount","Returns the number of members in a sorted set within a lexicographical range.","O(log(N)) with N being the number of elements in the sorted set.","2.8.9",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZLEXCOUNT_History,0,ZLEXCOUNT_Tips,0,zlexcountCommand,4,CMD_READONLY|CMD_FAST,ACL_CATEGORY_SORTEDSET,ZLEXCOUNT_Keyspecs,1,NULL,3),.args=ZLEXCOUNT_Args}, {MAKE_CMD("zmpop","Returns the highest- or lowest-scoring members from one or more sorted sets after removing them. Deletes the sorted set if the last member was popped.","O(K) + O(M*log(N)) where K is the number of provided keys, N being the number of elements in the sorted set, and M being the number of elements popped.","7.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZMPOP_History,0,ZMPOP_Tips,0,zmpopCommand,-4,CMD_WRITE,ACL_CATEGORY_SORTEDSET,ZMPOP_Keyspecs,1,zmpopGetKeys,4),.args=ZMPOP_Args}, {MAKE_CMD("zmscore","Returns the score of one or more members in a sorted set.","O(N) where N is the number of members being requested.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZMSCORE_History,0,ZMSCORE_Tips,0,zmscoreCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_SORTEDSET,ZMSCORE_Keyspecs,1,NULL,2),.args=ZMSCORE_Args}, @@ -12012,8 +12053,8 @@ struct COMMAND_STRUCT redisCommandTable[] = { {MAKE_CMD("zrevrank","Returns the index of a member in a sorted set ordered by descending scores.","O(log(N))","2.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZREVRANK_History,1,ZREVRANK_Tips,0,zrevrankCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_SORTEDSET,ZREVRANK_Keyspecs,1,NULL,3),.args=ZREVRANK_Args}, {MAKE_CMD("zscan","Iterates over members and scores of a sorted set.","O(1) for every call. O(N) for a complete iteration, including enough command calls for the cursor to return back to 0. N is the number of elements inside the collection.","2.8.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZSCAN_History,0,ZSCAN_Tips,1,zscanCommand,-3,CMD_READONLY,ACL_CATEGORY_SORTEDSET,ZSCAN_Keyspecs,1,NULL,4),.args=ZSCAN_Args}, {MAKE_CMD("zscore","Returns the score of a member in a sorted set.","O(1)","1.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZSCORE_History,0,ZSCORE_Tips,0,zscoreCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_SORTEDSET,ZSCORE_Keyspecs,1,NULL,2),.args=ZSCORE_Args}, -{MAKE_CMD("zunion","Returns the union of multiple sorted sets.","O(N)+O(M*log(M)) with N being the sum of the sizes of the input sorted sets, and M being the number of elements in the resulting sorted set.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZUNION_History,0,ZUNION_Tips,0,zunionCommand,-3,CMD_READONLY,ACL_CATEGORY_SORTEDSET,ZUNION_Keyspecs,1,zunionInterDiffGetKeys,5),.args=ZUNION_Args}, -{MAKE_CMD("zunionstore","Stores the union of multiple sorted sets in a key.","O(N)+O(M log(M)) with N being the sum of the sizes of the input sorted sets, and M being the number of elements in the resulting sorted set.","2.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZUNIONSTORE_History,0,ZUNIONSTORE_Tips,0,zunionstoreCommand,-4,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SORTEDSET,ZUNIONSTORE_Keyspecs,2,zunionInterDiffStoreGetKeys,5),.args=ZUNIONSTORE_Args}, +{MAKE_CMD("zunion","Returns the union of multiple sorted sets.","O(N)+O(M*log(M)) with N being the sum of the sizes of the input sorted sets, and M being the number of elements in the resulting sorted set.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZUNION_History,1,ZUNION_Tips,0,zunionCommand,-3,CMD_READONLY,ACL_CATEGORY_SORTEDSET,ZUNION_Keyspecs,1,zunionInterDiffGetKeys,5),.args=ZUNION_Args}, +{MAKE_CMD("zunionstore","Stores the union of multiple sorted sets in a key.","O(N)+O(M log(M)) with N being the sum of the sizes of the input sorted sets, and M being the number of elements in the resulting sorted set.","2.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZUNIONSTORE_History,1,ZUNIONSTORE_Tips,0,zunionstoreCommand,-4,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SORTEDSET,ZUNIONSTORE_Keyspecs,2,zunionInterDiffStoreGetKeys,5),.args=ZUNIONSTORE_Args}, /* stream */ {MAKE_CMD("xack","Returns the number of messages that were successfully acknowledged by the consumer group member of a stream.","O(1) for each message ID processed.","5.0.0",CMD_DOC_NONE,NULL,NULL,"stream",COMMAND_GROUP_STREAM,XACK_History,0,XACK_Tips,0,xackCommand,-4,CMD_WRITE|CMD_FAST,ACL_CATEGORY_STREAM,XACK_Keyspecs,1,NULL,3),.args=XACK_Args}, {MAKE_CMD("xackdel","Acknowledges and deletes one or multiple messages for a stream consumer group.","O(1) for each message ID processed.","8.2.0",CMD_DOC_NONE,NULL,NULL,"stream",COMMAND_GROUP_STREAM,XACKDEL_History,0,XACKDEL_Tips,0,xackdelCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_STREAM,XACKDEL_Keyspecs,1,NULL,4),.args=XACKDEL_Args}, @@ -12041,7 +12082,6 @@ struct COMMAND_STRUCT redisCommandTable[] = { {MAKE_CMD("decrby","Decrements a number from the integer value of a key. Uses 0 as initial value if the key doesn't exist.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,DECRBY_History,0,DECRBY_Tips,0,decrbyCommand,3,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_STRING,DECRBY_Keyspecs,1,NULL,2),.args=DECRBY_Args}, {MAKE_CMD("delex","Conditionally removes the specified key based on value or digest comparison.","O(1) for IFEQ/IFNE, O(N) for IFDEQ/IFDNE where N is the length of the string value.","8.4.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,DELEX_History,0,DELEX_Tips,0,delexCommand,-2,CMD_WRITE|CMD_FAST,ACL_CATEGORY_STRING,DELEX_Keyspecs,1,delexGetKeys,2),.args=DELEX_Args}, {MAKE_CMD("digest","Returns the XXH3 hash of a string value.","O(N) where N is the length of the string value.","8.4.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,DIGEST_History,0,DIGEST_Tips,0,digestCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_STRING,DIGEST_Keyspecs,1,NULL,1),.args=DIGEST_Args}, -{MAKE_CMD("gcra","Rate limit via GCRA (Generic Cell Rate Algorithm).","O(1)","8.8.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,GCRA_History,0,GCRA_Tips,0,gcraCommand,-5,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_STRING,GCRA_Keyspecs,1,NULL,5),.args=GCRA_Args}, {MAKE_CMD("get","Returns the string value of a key.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,GET_History,0,GET_Tips,0,getCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_STRING,GET_Keyspecs,1,NULL,1),.args=GET_Args}, {MAKE_CMD("getdel","Returns the string value of a key after deleting the key.","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,GETDEL_History,0,GETDEL_Tips,0,getdelCommand,2,CMD_WRITE|CMD_FAST,ACL_CATEGORY_STRING,GETDEL_Keyspecs,1,NULL,1),.args=GETDEL_Args}, {MAKE_CMD("getex","Returns the string value of a key after setting its expiration time.","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,GETEX_History,0,GETEX_Tips,0,getexCommand,-2,CMD_WRITE|CMD_FAST,ACL_CATEGORY_STRING,GETEX_Keyspecs,1,NULL,2),.args=GETEX_Args}, diff --git a/src/commands/gcra.json b/src/commands/gcra.json index cc0e029c2..6980af1ac 100644 --- a/src/commands/gcra.json +++ b/src/commands/gcra.json @@ -2,7 +2,7 @@ "GCRA": { "summary": "Rate limit via GCRA (Generic Cell Rate Algorithm).", "complexity": "O(1)", - "group": "string", + "group": "rate_limit", "since": "8.8.0", "arity": -5, "function": "gcraCommand", @@ -12,7 +12,7 @@ "FAST" ], "acl_categories": [ - "STRING" + "RATE_LIMIT" ], "key_specs": [ { diff --git a/src/commands/gcrasetvalue.json b/src/commands/gcrasetvalue.json new file mode 100644 index 000000000..5cce15cf4 --- /dev/null +++ b/src/commands/gcrasetvalue.json @@ -0,0 +1,52 @@ +{ + "GCRASETVALUE": { + "summary": "An internal command for recording a GCRA TAT value during AOF rewrite and replication.", + "complexity": "O(1)", + "group": "rate_limit", + "since": "8.8.0", + "arity": 3, + "function": "gcraSetValueCommand", + "command_flags": [ + "WRITE", + "DENYOOM", + "FAST" + ], + "acl_categories": [ + "RATE_LIMIT" + ], + "key_specs": [ + { + "flags": [ + "OW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "const": "OK" + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "tat", + "type": "integer" + } + ] + } +} diff --git a/src/commands/zinter.json b/src/commands/zinter.json index 4828e21d6..1b192cdb2 100644 --- a/src/commands/zinter.json +++ b/src/commands/zinter.json @@ -7,6 +7,12 @@ "arity": -3, "function": "zinterCommand", "get_keys_function": "zunionInterDiffGetKeys", + "history": [ + [ + "8.8.0", + "Added `COUNT` aggregate option." + ] + ], "command_flags": [ "READONLY" ], @@ -101,6 +107,12 @@ "name": "max", "type": "pure-token", "token": "MAX" + }, + { + "name": "count", + "type": "pure-token", + "token": "COUNT", + "since": "8.8.0" } ] }, diff --git a/src/commands/zinterstore.json b/src/commands/zinterstore.json index 5bd940c65..0404bf749 100644 --- a/src/commands/zinterstore.json +++ b/src/commands/zinterstore.json @@ -7,6 +7,12 @@ "arity": -4, "function": "zinterstoreCommand", "get_keys_function": "zunionInterDiffStoreGetKeys", + "history": [ + [ + "8.8.0", + "Added `COUNT` aggregate option." + ] + ], "command_flags": [ "WRITE", "DENYOOM" @@ -100,6 +106,12 @@ "name": "max", "type": "pure-token", "token": "MAX" + }, + { + "name": "count", + "type": "pure-token", + "token": "COUNT", + "since": "8.8.0" } ] } diff --git a/src/commands/zunion.json b/src/commands/zunion.json index 1ce3dc5ee..366e0e8f9 100644 --- a/src/commands/zunion.json +++ b/src/commands/zunion.json @@ -7,6 +7,12 @@ "arity": -3, "function": "zunionCommand", "get_keys_function": "zunionInterDiffGetKeys", + "history": [ + [ + "8.8.0", + "Added `COUNT` aggregate option." + ] + ], "command_flags": [ "READONLY" ], @@ -101,6 +107,12 @@ "name": "max", "type": "pure-token", "token": "MAX" + }, + { + "name": "count", + "type": "pure-token", + "token": "COUNT", + "since": "8.8.0" } ] }, diff --git a/src/commands/zunionstore.json b/src/commands/zunionstore.json index 65e7b5469..fd208a6c0 100644 --- a/src/commands/zunionstore.json +++ b/src/commands/zunionstore.json @@ -7,6 +7,12 @@ "arity": -4, "function": "zunionstoreCommand", "get_keys_function": "zunionInterDiffStoreGetKeys", + "history": [ + [ + "8.8.0", + "Added `COUNT` aggregate option." + ] + ], "command_flags": [ "WRITE", "DENYOOM" @@ -99,6 +105,12 @@ "name": "max", "type": "pure-token", "token": "MAX" + }, + { + "name": "count", + "type": "pure-token", + "token": "COUNT", + "since": "8.8.0" } ] } diff --git a/src/config.c b/src/config.c index e02cd64e5..0ad28ef5b 100644 --- a/src/config.c +++ b/src/config.c @@ -2917,7 +2917,7 @@ static int setConfigNotifyKeyspaceEventsOption(standardConfig *config, sds *argv } int flags = keyspaceEventsStringToFlags(argv[0]); if (flags == -1) { - *err = "Invalid event class character. Use 'Ag$lshzxeKEtmdn'."; + *err = "Invalid event class character. Use 'Ag$lshzxeKEtmdnocr'."; return 0; } server.notify_keyspace_events = flags; diff --git a/src/db.c b/src/db.c index 7d7a3e96b..32c058dab 100644 --- a/src/db.c +++ b/src/db.c @@ -1083,7 +1083,7 @@ redisDb *initTempDb(void) { tempDb[i].expires = kvstoreCreate(&kvstoreBaseType, &dbExpiresDictType, slot_count_bits, flags); tempDb[i].subexpires = estoreCreate(&subexpiresBucketsType, slot_count_bits); - tempDb[i].stream_idmp_keys = dictCreate(&objectKeyPointerValueDictType); + tempDb[i].stream_idmp_keys = dictCreate(&objectKeyNoValueDictType); } return tempDb; @@ -1117,7 +1117,7 @@ void streamMoveIdmpKeys(dict *src, dict *dst, int slot) { while ((de = dictNext(di)) != NULL) { robj *key = dictGetKey(de); if (calculateKeySlot(key->ptr) == slot) { - if (dictAdd(dst, key, dictGetVal(de)) == DICT_OK) { + if (dictAddRaw(dst, key, NULL)) { incrRefCount(key); } dictDelete(src, key); @@ -1756,7 +1756,8 @@ char *obj_type_name[OBJ_TYPE_MAX] = { "zset", "hash", NULL, /* module type is special */ - "stream" + "stream", + "gcra" }; /* Helper function to get type from a string in scan commands */ @@ -2438,6 +2439,7 @@ void copyCommand(client *c) { case OBJ_ZSET: newobj = zsetDup(o); break; case OBJ_HASH: newobj = hashTypeDup(o, &minHashExpire); break; case OBJ_STREAM: newobj = streamDup(o); break; + case OBJ_GCRA: newobj = gcraDup(o); break; case OBJ_MODULE: newobj = moduleTypeDupOrReply(c, key, newkey, dst->id, o); if (!newobj) return; diff --git a/src/debug.c b/src/debug.c index 29ae88298..c6baf4b4d 100644 --- a/src/debug.c +++ b/src/debug.c @@ -123,6 +123,14 @@ void mixStringObjectDigest(unsigned char *digest, robj *o) { decrRefCount(o); } +void mixGCRAObjectDigest(unsigned char *digest, robj *o) { + char buf[LONG_STR_SIZE]; + long long val; + getLongLongFromGCRAObject(o, &val); + int len = ll2string(buf, sizeof(buf), val); + mixDigest(digest,buf,len); +} + /* This function computes the digest of a data structure stored in the * object 'o'. It is the core of the DEBUG DIGEST command: when taking the * digest of a whole dataset, we take the digest of the key and the value @@ -255,6 +263,8 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o) } } streamIteratorStop(&si); + } else if (o->type == OBJ_GCRA) { + mixGCRAObjectDigest(digest, o); } else if (o->type == OBJ_MODULE) { RedisModuleDigest md = {{0},{0},keyobj,db->id}; moduleValue *mv = o->ptr; @@ -896,7 +906,7 @@ NULL addReplyError(c,"Wrong protocol type name. Please use one of the following: string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false"); } } else if (!strcasecmp(c->argv[1]->ptr,"sleep") && c->argc == 3) { - double dtime = fast_float_strtod(c->argv[2]->ptr,NULL); + double dtime = fast_float_strtod(c->argv[2]->ptr,sdslen(c->argv[2]->ptr),NULL); long long utime = dtime*1000000; struct timespec tv; @@ -1302,6 +1312,10 @@ void serverLogObjectDebugInfo(const robj *o) { serverLog(LL_WARNING,"Skiplist level: %d", (int) ((const zset*)o->ptr)->zsl->level); } else if (o->type == OBJ_STREAM) { serverLog(LL_WARNING,"Stream size: %d", (int) streamLength(o)); + } else if (o->type == OBJ_GCRA) { +#if UINTPTR_MAX == 0xffffffffffffffff + serverLog(LL_WARNING, "GCRA object: %lld", (long long)o->ptr); +#endif } #endif } diff --git a/src/defrag.c b/src/defrag.c index b6433a2e7..ae1e632d8 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -1211,6 +1211,13 @@ void defragKey(defragKeysCtx *ctx, dictEntry *de, dictEntryLink link) { } } else if (ob->type == OBJ_STREAM) { defragStream(ctx, ob); + } else if (ob->type == OBJ_GCRA) { + /* GCRA object is just an allocation to a long long value */ +#if UINTPTR_MAX == 0xffffffff + void *newptr, *ptr = ob->ptr; + if ((newptr = activeDefragAlloc(ptr))) + ob->ptr = newptr; +#endif } else if (ob->type == OBJ_MODULE) { defragModule(ctx,db, ob); } else { diff --git a/src/fast_float_strtod.c b/src/fast_float_strtod.c new file mode 100644 index 000000000..48a5df502 --- /dev/null +++ b/src/fast_float_strtod.c @@ -0,0 +1,544 @@ +/* fast_float_strtod.c - Fast string to double conversion + * + * This is a C conversion of a subset of the fast_float C++ library, + * implementing only what Redis needs: parsing decimal floating-point strings. + * + * Original fast_float library: + * https://github.com/fastfloat/fast_float + * by Daniel Lemire and João Paulo Magalhaes + * + * MIT License + * + * Copyright (c) 2021 The fast_float authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +#include "fast_float_strtod.h" +#include "config.h" +#include "zmalloc.h" + +/* Powers of 10 from 10^0 to 10^22 (exact in double precision). + * These are the only powers of 10 that can be exactly represented as doubles. */ +static const double powers_of_ten[] = { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 +}; + +/* Maximum mantissa for fast path: 2^53 */ +#define MAX_MANTISSA_FAST_PATH 9007199254740992ULL /* 2^53 */ + +/* Exponent limits for fast path */ +#define MIN_EXPONENT_FAST_PATH -22 +#define MAX_EXPONENT_FAST_PATH 22 + +/* Maximum number of significant digits we track before overflow */ +#define MAX_DIGITS 19 + +/* Case-insensitive match against known lowercase literals using `| 0x20`. + * Only valid when the target characters are ASCII letters (a-z). */ +static inline int strcasecmp_3(const char *s, char c0, char c1, char c2) { + return ((s[0] | 0x20) == c0) & ((s[1] | 0x20) == c1) & ((s[2] | 0x20) == c2); +} + +/* Case-insensitive comparison for first n characters. + * Only valid when the target characters are ASCII letters (a-z). */ +static int strncasecmp_local(const char *s1, const char *s2, size_t n) { + for (size_t i = 0; i < n; i++) { + int diff = (s1[i] | 0x20) - s2[i]; + if (diff) return diff; + } + return 0; +} + +/* Parse inf/nan special values. + * Returns 1 if parsed successfully, 0 otherwise. + * On success, *endptr points past the parsed value. */ +static inline int parse_infnan(const char *p, const char *pend, double *result, const char **endptr) { + int negative = (*p == '-'); + if (*p == '-' || *p == '+') p++; + size_t remaining = pend - p; + + if (remaining >= 3) { + if (strcasecmp_3(p, 'n', 'a', 'n')) { + *result = negative ? -NAN : NAN; + p += 3; + /* Check for optional nan(n-char-seq) */ + if (p < pend && *p == '(') { + const char *start = p; + p++; + while (p < pend) { + char c = *p; + if (c == ')') { + p++; + break; + } + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_')) { + /* Invalid character, revert to position after "nan" */ + p = start; + break; + } + p++; + } + /* If we didn't find closing ')', revert */ + if (p[-1] != ')') { + p = start; + } + } + if (endptr) *endptr = (char *)p; + return 1; + } + if (strcasecmp_3(p, 'i', 'n', 'f')) { + *result = negative ? -INFINITY : INFINITY; + p += 3; + /* Check for optional "inity" suffix */ + if (remaining == 8 && strncasecmp_local(p, "inity", 5) == 0) { + p += 5; + } + if (endptr) *endptr = (char *)p; + return 1; + } + } + return 0; +} + +/* SWAR (SIMD Within A Register) helpers for batch digit parsing. */ + +static inline uint64_t read8_to_u64(const char *p) { + uint64_t val; + memcpy(&val, p, sizeof(uint64_t)); +#if BYTE_ORDER == BIG_ENDIAN + /* SWAR digit parsing assumes first char in LSB (little-endian layout). */ +#if defined(__GNUC__) || defined(__clang__) + val = __builtin_bswap64(val); +#else + val = ((val & 0x00000000FFFFFFFFULL) << 32) | ((val & 0xFFFFFFFF00000000ULL) >> 32); + val = ((val & 0x0000FFFF0000FFFFULL) << 16) | ((val & 0xFFFF0000FFFF0000ULL) >> 16); + val = ((val & 0x00FF00FF00FF00FFULL) << 8) | ((val & 0xFF00FF00FF00FF00ULL) >> 8); +#endif +#endif + return val; +} + +static inline int is_made_of_eight_digits(uint64_t val) { + return !((((val + 0x4646464646464646ULL) | (val - 0x3030303030303030ULL)) & + 0x8080808080808080ULL)); +} + +static inline uint32_t parse_eight_digits_swar(uint64_t val) { + uint64_t const mask = 0x000000FF000000FFULL; + uint64_t const mul1 = 0x000F424000000064ULL; /* 100 + (1000000ULL << 32) */ + uint64_t const mul2 = 0x0000271000000001ULL; /* 1 + (10000ULL << 32) */ + val -= 0x3030303030303030ULL; + val = (val * 10) + (val >> 8); + val = (((val & mask) * mul1) + (((val >> 16) & mask) * mul2)) >> 32; + return (uint32_t)val; +} + +/* Parse a decimal number string into components. + * This follows the fast_float algorithm closely. */ +static inline int parse_number_string(const char *p, const char *pend, double *result, const char **endptr) { + uint64_t mantissa = 0; /* Mantissa digits as uint64 */ + int64_t exponent = 0; /* Decimal exponent (adjusted for decimal point) */ + int negative = 0; /* Sign flag */ + *endptr = p; + + if (p == pend) return 0; + + /* Parse sign */ + negative = (*p == '-'); + if (*p == '-' || *p == '+') { + p++; + if (p == pend) return 0; + } + + const char *start_digits = p; + + /* Parse integer part */ + mantissa = 0; + while (pend - p >= 8) { + uint64_t val = read8_to_u64(p); + if (!is_made_of_eight_digits(val)) break; + mantissa = mantissa * 100000000 + parse_eight_digits_swar(val); + p += 8; + } + while (p != pend && *p >= '0' && *p <= '9') { + mantissa = mantissa * 10 + (*p - '0'); + p++; + } + + int64_t digit_count = p - start_digits; + + /* Parse decimal point and fractional part */ + exponent = 0; + int has_decimal = (p != pend && *p == '.'); + + if (has_decimal) { + p++; + const char *before = p; + while (pend - p >= 8) { + uint64_t val = read8_to_u64(p); + if (!is_made_of_eight_digits(val)) break; + mantissa = mantissa * 100000000 + parse_eight_digits_swar(val); + p += 8; + } + while (p != pend && *p >= '0' && *p <= '9') { + mantissa = mantissa * 10 + (*p - '0'); + p++; + } + exponent = before - p; /* Negative: number of fractional digits */ + digit_count += (p - before); + } + + /* Must have at least one digit */ + if (digit_count == 0) return 0; + + /* Parse exponent */ + int64_t exp_number = 0; + if (p != pend && (*p == 'e' || *p == 'E')) { + const char *exp_start = p; + p++; + + int neg_exp = 0; + if (p != pend && *p == '-') { + neg_exp = 1; + p++; + } else if (p != pend && *p == '+') { + p++; + } + + if (p == pend || *p < '0' || *p > '9') { + /* No digits after e/E, revert to position before 'e' */ + p = exp_start; + } else { + while (p != pend && *p >= '0' && *p <= '9') { + if (exp_number < 0x10000000) { + exp_number = exp_number * 10 + (*p - '0'); + } + p++; + } + if (neg_exp) exp_number = -exp_number; + exponent += exp_number; + } + } + + *endptr = p; + + /* Handle overflow in mantissa: if we have too many digits, + * we need to reparse more carefully */ + if (digit_count > MAX_DIGITS) { + /* Skip leading zeros to get actual digit count */ + const char *s = start_digits; + while (s != pend && (*s == '0' || *s == '.')) { + if (*s == '0') digit_count--; + s++; + } + + if (digit_count > MAX_DIGITS) return 0; + } + + /* Check if we're within fast path bounds */ + if (exponent < MIN_EXPONENT_FAST_PATH) return 0; + if (exponent > MAX_EXPONENT_FAST_PATH) return 0; + if (mantissa > MAX_MANTISSA_FAST_PATH) return 0; + + /* Fast path: direct conversion */ + double value = (double)mantissa; + + if (exponent < 0) { + value = value / powers_of_ten[-exponent]; + } else if (exponent > 0) { + value = value * powers_of_ten[exponent]; + } + + if (negative) { + value = -value; + } + + *result = value; + return 1; +} + +/* Main conversion function. + * + * This function behaves similarly to the standard strtod function, converting + * the initial portion of the string pointed to by `nptr` to a `double` value. + * If the conversion fails, errno is set to EINVAL error code. + * + * @param nptr A pointer to the null-terminated byte string to be interpreted. + * @param endptr A pointer to a pointer to character. If `endptr` is not NULL, + * it will point to the character after the last character used + * in the conversion. + * @return The converted value as a double. If no valid conversion could + * be performed, returns 0.0. + */ +static inline int fast_float_try_fast(const char *nptr, const char *pend, double *result, const char **endptr) { + if (nptr == pend) { + errno = EINVAL; + if (endptr) *endptr = (char *)nptr; + return 0; + } + + /* Parse the number string */ + if (parse_number_string(nptr, pend, result, endptr)) { + return 1; + } + + /* Not a valid decimal number, try inf/nan special values */ + if (parse_infnan(nptr, pend, result, endptr)) { + return 1; + } + + return 0; +} + +static double fast_float_strtod_fallback(const char *nptr, size_t len, char **endptr) { + /* Since the input may not be null-terminated, we must copy it into a temporary buffer. */ + char static_buf[128]; + char *buf = static_buf; + if (len >= sizeof(static_buf)) + buf = zmalloc(len + 1); + memcpy(buf, nptr, len); + buf[len] = '\0'; + + char *fallback_end; + double result = strtod(buf, &fallback_end); + if (endptr) *endptr = (char *)nptr + (fallback_end - buf); + + /* If strtod failed to parse, set errno */ + if (fallback_end == buf) { + errno = EINVAL; + } + + if (buf != static_buf) zfree(buf); + return result; +} + +/* Convert string to double, with explicit length (string need NOT be null-terminated). + * Falls back to strtod by copying to a temporary null-terminated buffer. */ +double fast_float_strtod(const char *nptr, size_t len, char **endptr) { + double result = 0.0; + const char *pend = nptr + len; + const char *eptr; + + /* Use fast path for non-null-terminated strings */ + if (likely(fast_float_try_fast(nptr, pend, &result, &eptr) && eptr == pend)) { + if (endptr) *endptr = (char *)eptr; +#if UINTPTR_MAX == 0xffffffff + /* On 32-bit x86 with x87 FPU, the fast-path fdiv/fmul result lives in + * an 80-bit extended-precision register. With optimisation the compiler + * may return that value in st(0) without ever storing it to a 64-bit + * memory slot, so the caller would receive an 80-bit value that differs + * from the correctly-rounded 64-bit double. Writing through a volatile + * forces a real fstpl (store + pop to 64-bit memory) followed by fldl + * (reload into st(0) from that 64-bit slot), ensuring the return value + * is truncated to double precision before it reaches the caller. */ + volatile double ret = result; + return ret; +#else + return result; +#endif + } + + /* Fall back to strtod for complex cases: + * - Very large or very small exponents + * - Too many digits (need precise rounding) + * This ensures we get correctly-rounded results for edge cases. */ + return fast_float_strtod_fallback(nptr, len, endptr); +} + +#ifdef REDIS_TEST +#include +#include "testhelp.h" + +#define UNUSED(x) (void)(x) +#define COUNTOF(arr) (int)(sizeof(arr) / sizeof((arr)[0])) + +typedef struct { + const char *input; + double expected; +} ff_testcase; + +static int ff_eq(double a, double b) { + if (isnan(a)) return isnan(b); + if (isinf(a)) return isinf(b) && (a > 0) == (b > 0); + return a == b; +} + +static void run_ff_tests(ff_testcase *cases, int n, int expect_failed) { + for (int i = 0; i < n; i++) { + const char *s = cases[i].input; + size_t len = strlen(s); + char *eptr; + + errno = 0; + double d = fast_float_strtod(s, len, &eptr); + int failed = ((size_t)(eptr - s) != len) || errno == EINVAL || + (errno == ERANGE && (d == HUGE_VAL || d == -HUGE_VAL || fpclassify(d) == FP_ZERO)); + int ok = (expect_failed == failed) && ff_eq(d, cases[i].expected); + char descr[128]; + if (ok) + snprintf(descr, sizeof(descr), "\"%s\" -> expect %s(%.20g)", + s, expect_failed ? "fail" : "ok", cases[i].expected); + else + snprintf(descr, sizeof(descr), "\"%s\" -> expect %s(%.20g) but got %s(%.20g)", + s, expect_failed ? "fail" : "ok", cases[i].expected, failed ? "fail" : "ok", d); + test_cond(descr, ok); + } +} + +int fastFloatTest(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); + UNUSED(flags); + + /* Finite decimals: fast path, exponent ±22 edges, mantissa 2^53, strtod fallback. */ + ff_testcase decimal_ok[] = { + {"0", 0.0}, + {"+0", 0.0}, + {"-0", -0.0}, + {"42", 42.0}, + {"+42", 42.0}, + {"-42", -42.0}, + {"00007", 7.0}, + {"00.25", 0.25}, + {"3.14", 3.14}, + {".5", 0.5}, + {"+.5", 0.5}, + {"1.", 1.0}, + {"0.", 0.0}, + {".0", 0.0}, + {"-1.5e2", -150.0}, + {"1e5", 1e5}, + {"1E5", 1e5}, + {"2E3", 2000.0}, + {"3e+5", 3e5}, + {"1e-10", 1e-10}, + {"1e-22", 1e-22}, + {"1e+22", 1e22}, + {"1e-23", 1e-23}, + {"1e+100", 1e100}, + {"1e-100", 1e-100}, + {"9007199254740992", 9007199254740992.0}, + {"9007199254740993", 9007199254740992.0}, + {"12345678901234567890", 1.2345678901234567e19}, + {"2.2250738585072012e-308", 2.2250738585072012e-308}, /* Near DBL_MIN boundary */ + {"0x10", 16.0}, + }; + run_ff_tests(decimal_ok, COUNTOF(decimal_ok), 0); + + /* No valid prefix for full buffer, or trailing junk. */ + ff_testcase decimal_bad[] = { + {"1abc", 1.0}, + {"1e", 1.0}, + {"1e+", 1.0}, + {"1e-", 1.0}, + {"1e+z", 1.0}, + {"12.34.56", 12.34}, + {"..1", 0.0}, + {"e10", 0.0}, + {"E10", 0.0}, + {"+", 0.0}, + {"-", 0.0}, + {"foo", 0.0}, + {"1 ", 1.0}, + {"3.14!", 3.14}, + }; + run_ff_tests(decimal_bad, COUNTOF(decimal_bad), 1); + + ff_testcase inf_valid[] = { + {"inf", INFINITY}, + {"INF", INFINITY}, + {"Inf", INFINITY}, + {"infinity", INFINITY}, + {"INFINITY", INFINITY}, + {"Infinity", INFINITY}, + {"+inf", INFINITY}, + {"-inf", -INFINITY}, + {"+infinity", INFINITY}, + {"-INFINITY", -INFINITY}, + }; + run_ff_tests(inf_valid, COUNTOF(inf_valid), 0); + + ff_testcase inf_invalid[] = { + {"in", 0}, + {"infin", INFINITY}, + {"infini1", INFINITY}, + {"infinitx", INFINITY}, + {"infinityy", INFINITY}, + {"info", INFINITY}, + {"ina", 0}, + {"INFI", INFINITY}, + {"iNf0", INFINITY}, + }; + run_ff_tests(inf_invalid, COUNTOF(inf_invalid), 1); + + ff_testcase nan_valid[] = { + {"nan", NAN}, + {"NAN", NAN}, + {"Nan", NAN}, + {"nan(123)", NAN}, + {"nan(abc)", NAN}, + {"nan(123abc)", NAN}, + }; + run_ff_tests(nan_valid, COUNTOF(nan_valid), 0); + + ff_testcase nan_invalid[] = { + {"na", 0}, + {"nan(", NAN}, /* unclosed paren */ + {"nan(abc", NAN}, /* missing closing paren */ + {"nan(ab!c)", NAN}, /* invalid char in paren */ + {"nan(ab c)", NAN}, /* space in paren */ + {"nanx", NAN}, /* trailing garbage */ + }; + run_ff_tests(nan_invalid, COUNTOF(nan_invalid), 1); + + /* Large input that exceeds static_buf (128 bytes), exercising the zmalloc fallback path. */ + { + /* Build a string "000...00042.0" with total length > 128. */ + char big[256]; + memset(big, '0', sizeof(big)); + big[sizeof(big) - 4] = '2'; + big[sizeof(big) - 3] = '.'; + big[sizeof(big) - 2] = '0'; + big[sizeof(big) - 1] = '\0'; + char *eptr; + double d = fast_float_strtod(big, strlen(big), &eptr); + test_cond("large input (>128 bytes) zmalloc fallback path", + (size_t)(eptr - big) == strlen(big) && ff_eq(d, 2.0)); + + /* Large input that is completely invalid. */ + memset(big, 'x', sizeof(big) - 1); + big[sizeof(big) - 1] = '\0'; + d = fast_float_strtod(big, strlen(big), &eptr); + test_cond("invalid large input (>128 bytes) zmalloc fallback path", + eptr == big && ff_eq(d, 0.0)); + } + + return 0; +} +#endif diff --git a/src/fast_float_strtod.h b/src/fast_float_strtod.h new file mode 100644 index 000000000..91ab9cfbf --- /dev/null +++ b/src/fast_float_strtod.h @@ -0,0 +1,13 @@ + +#ifndef __FAST_FLOAT_STRTOD_H__ +#define __FAST_FLOAT_STRTOD_H__ + +#include + +double fast_float_strtod(const char *nptr, size_t len, char **endptr); + +#ifdef REDIS_TEST +int fastFloatTest(int argc, char **argv, int flags); +#endif + +#endif /* __FAST_FLOAT_STRTOD_H__ */ diff --git a/src/gcra.c b/src/gcra.c index 9a2c23df5..488fad5ce 100644 --- a/src/gcra.c +++ b/src/gcra.c @@ -129,23 +129,11 @@ void gcraCommand(client *c) { long long tat_us, new_tat_us; dictEntryLink link; kvobj *kv = lookupKeyWriteWithLink(c->db, key, &link); - if (checkType(c, kv, OBJ_STRING)) { + if (checkType(c, kv, OBJ_GCRA)) { return; } if (kv != NULL) { - /* Note the value of the key may have been overwritten outside of the - * GCRA command (f.e by calling SET). We don't try to catch such errors - * as this would be possible only with a dedicated structures for GCRA, - * while using STRING gives us all the benefits of a redis key - - * replication, setting expiration, etc. */ - if (getLongLongFromObject(kv, &tat_us) != C_OK) { - addReplyError(c, "Invalid GCRA key"); - return; - } - if (tat_us <= 0) { - addReplyError(c, "Negative time is invalid value for GCRA"); - return; - } + getLongLongFromGCRAObject(kv, &tat_us); } else { tat_us = now; } @@ -208,10 +196,18 @@ void gcraCommand(client *c) { } else { limited = 0; ttl_us = new_tat_us - now; - robj *tatobj = createStringObjectFromLongLong(new_tat_us); + robj *tatobj = createGCRAObject(new_tat_us); setKeyByLink(c, c->db, key, &tatobj, kv ? SETKEY_ALREADY_EXIST : SETKEY_DOESNT_EXIST, &link); - notifyKeyspaceEvent(NOTIFY_STRING,"set",key,c->db->id); + notifyKeyspaceEvent(NOTIFY_RATE_LIMIT,"gcra",key,c->db->id); + /* The key implicitly sets its own expiry time (which is basically the + * TaT after which time the value is no longer of any use). That way even + * if only one GCRA command is called on a key it will automatically + * expire after reaching its TaT without user needing to explicitly call + * DEL on it. + * These keys are expected to be numerous and short lived thus the + * decision to keep the implicit expiraty. + * NOTE: idea is same as in redis-cell. */ long long when = new_tat_us / 1000; kv = setExpireByLink(c, c->db, key->ptr, when, link); notifyKeyspaceEvent(NOTIFY_GENERIC,"expire",key,c->db->id); @@ -219,9 +215,11 @@ void gcraCommand(client *c) { /* Replicating the command directly would mess up TaT as we use * commandTimeSnapshot. We instead rewrite the command as SET with the * appropriate expire time. */ - robj *pexat_obj = createStringObjectFromLongLong(when); - rewriteClientCommandVector(c, 5, shared.set, key, kv, shared.pxat, pexat_obj); - decrRefCount(pexat_obj); + robj *gcrasetvalue = createStringObject("GCRASETVALUE", 12); + robj *newtatstr = createStringObjectFromLongLong(new_tat_us); + rewriteClientCommandVector(c, 3, gcrasetvalue, key, newtatstr); + decrRefCount(gcrasetvalue); + decrRefCount(newtatstr); server.dirty++; } @@ -239,3 +237,44 @@ void gcraCommand(client *c) { addReplyLongLong(c, retry_after_s); addReplyLongLong(c, reset_after_s); } + +/* GCRASETVALUE key tat + * + * Internal command used during AOF rewrite to record a GCRA TAT value. The GCRA + * command is also rewritten as GCRASETVALUE for replication since GCRA uses + * commandTimeSnapshot. */ +void gcraSetValueCommand(client *c) { + robj *key = c->argv[1]; + robj *tat = c->argv[2]; + long long when; + + dictEntryLink link; + kvobj *kv = lookupKeyWriteWithLink(c->db, key, &link); + if (checkType(c, kv, OBJ_GCRA)) return; + + if (getLongLongFromObjectOrReply(c, tat, &when, "Invalid TaT value") == C_ERR) { + return; + } + if (when < 0) { + addReplyError(c, "Invalid negative TaT value"); + return; + } + + robj *tatobj = createGCRAObject(when); + setKeyByLink(c, c->db, key, &tatobj, kv ? SETKEY_ALREADY_EXIST : SETKEY_DOESNT_EXIST, &link); + notifyKeyspaceEvent(NOTIFY_RATE_LIMIT,"gcra",key,c->db->id); + + /* Just like the base GCRA command we set the expire time of the key implicitly. */ + long long when_ms = when / 1000; + kv = setExpireByLink(c, c->db, key->ptr, when_ms, link); + notifyKeyspaceEvent(NOTIFY_GENERIC,"expire",key,c->db->id); + server.dirty++; + + addReply(c, shared.ok); +} + +robj *gcraDup(robj *o) { + long long val; + getLongLongFromGCRAObject(o, &val); + return createGCRAObject(val); +} diff --git a/src/lazyfree.c b/src/lazyfree.c index 87e716233..01f34bec6 100644 --- a/src/lazyfree.c +++ b/src/lazyfree.c @@ -332,7 +332,7 @@ void emptyDbAsync(redisDb *db) { db->keys = kvstoreCreate(&kvstoreExType, &dbDictType, slot_count_bits, flags); db->expires = kvstoreCreate(&kvstoreBaseType, &dbExpiresDictType, slot_count_bits, flags); db->subexpires = estoreCreate(&subexpiresBucketsType, slot_count_bits); - db->stream_idmp_keys = dictCreate(&objectKeyPointerValueDictType); + db->stream_idmp_keys = dictCreate(&objectKeyNoValueDictType); protectClientReplyObjects(); /* Protect client reply objects before async free. */ emptyDbDataAsync(oldkeys, oldexpires, oldsubexpires, old_stream_idmp_keys, NULL); } diff --git a/src/module.c b/src/module.c index fc28b37a1..d4a857a2e 100644 --- a/src/module.c +++ b/src/module.c @@ -4225,6 +4225,7 @@ int RM_KeyType(RedisModuleKey *key) { case OBJ_HASH: return REDISMODULE_KEYTYPE_HASH; case OBJ_MODULE: return REDISMODULE_KEYTYPE_MODULE; case OBJ_STREAM: return REDISMODULE_KEYTYPE_STREAM; + case OBJ_GCRA: return REDISMODULE_KEYTYPE_GCRA; default: return REDISMODULE_KEYTYPE_EMPTY; } } @@ -9202,10 +9203,11 @@ void moduleReleaseGIL(void) { * - REDISMODULE_NOTIFY_OVERWRITTEN: Overwritten events * - REDISMODULE_NOTIFY_TYPE_CHANGED: Type-changed events * - REDISMODULE_NOTIFY_KEY_TRIMMED: Key trimmed events after a slot migration operation + * - REDISMODULE_NOTIFY_RATE_LIMIT: Rate limit event * - REDISMODULE_NOTIFY_ALL: All events (Excluding REDISMODULE_NOTIFY_KEYMISS, * REDISMODULE_NOTIFY_NEW, REDISMODULE_NOTIFY_OVERWRITTEN, - * REDISMODULE_NOTIFY_TYPE_CHANGED - * and REDISMODULE_NOTIFY_KEY_TRIMMED) + * REDISMODULE_NOTIFY_TYPE_CHANGED, REDISMODULE_NOTIFY_KEY_TRIMMED + * and REDISMODULE_NOTIFY_RATE_LIMIT) * - REDISMODULE_NOTIFY_LOADED: A special notification available only for modules, * indicates that the key was loaded from persistence. * Notice, when this event fires, the given key diff --git a/src/notify.c b/src/notify.c index 00dd4a090..11ea53241 100644 --- a/src/notify.c +++ b/src/notify.c @@ -40,6 +40,7 @@ int keyspaceEventsStringToFlags(char *classes) { case 'n': flags |= NOTIFY_NEW; break; case 'o': flags |= NOTIFY_OVERWRITTEN; break; case 'c': flags |= NOTIFY_TYPE_CHANGED; break; + case 'r': flags |= NOTIFY_RATE_LIMIT; break; default: return -1; } } @@ -70,6 +71,7 @@ sds keyspaceEventsFlagsToString(int flags) { if (flags & NOTIFY_NEW) res = sdscatlen(res,"n",1); if (flags & NOTIFY_OVERWRITTEN) res = sdscatlen(res,"o",1); if (flags & NOTIFY_TYPE_CHANGED) res = sdscatlen(res,"c",1); + if (flags & NOTIFY_RATE_LIMIT) res = sdscatlen(res,"r",1); } if (flags & NOTIFY_KEYSPACE) res = sdscatlen(res,"K",1); if (flags & NOTIFY_KEYEVENT) res = sdscatlen(res,"E",1); diff --git a/src/object.c b/src/object.c index 9a9e6c257..cfcfa4844 100644 --- a/src/object.c +++ b/src/object.c @@ -514,6 +514,23 @@ robj *createStreamObject(void) { return o; } +robj *createGCRAObject(long long value) { + /* NOTE: for 32-bit systems we can't use integer encoding (as OBJ_STRING does) + * as the GCRA object is a unixtime value in microseconds, which as of the + * time of writing is already much more than 32-bit's LONG_MAX. */ +#if UINTPTR_MAX == 0xffffffff + long long *v = zmalloc(sizeof(long long)); + *v = value; + robj *o = createObject(OBJ_GCRA,v); +#else + robj *o = createObject(OBJ_GCRA,NULL); + o->ptr = (void*)value; +#endif + + o->encoding = OBJ_ENCODING_INT; + return o; +} + robj *createModuleObject(moduleType *mt, void *value) { moduleValue *mv = zmalloc(sizeof(*mv)); mv->type = mt; @@ -586,6 +603,14 @@ void freeStreamObject(robj *o) { freeStream(o->ptr); } +void freeGCRAObject(robj *o) { +#if UINTPTR_MAX == 0xffffffff + zfree(o->ptr); +#else + (void)o; +#endif +} + void incrRefCount(robj *o) { if (o->refcount < OBJ_FIRST_SPECIAL_REFCOUNT - 1) { o->refcount++; @@ -629,6 +654,7 @@ void decrRefCount(robj *o) { case OBJ_HASH: freeHashObject(o); break; case OBJ_MODULE: freeModuleObject(o); break; case OBJ_STREAM: freeStreamObject(o); break; + case OBJ_GCRA: freeGCRAObject(o); break; default: serverPanic("Unknown object type"); break; } } @@ -691,8 +717,7 @@ void dismissSetObject(robj *o, size_t size_hint) { } /* Dismiss hash table memory. */ - dismissMemory(set->ht_table[0], DICTHT_SIZE(set->ht_size_exp[0])*sizeof(dictEntry*)); - dismissMemory(set->ht_table[1], DICTHT_SIZE(set->ht_size_exp[1])*sizeof(dictEntry*)); + dismissDictBucketsMemory(set); } else if (o->encoding == OBJ_ENCODING_INTSET) { dismissMemory(o->ptr, intsetBlobLen((intset*)o->ptr)); } else if (o->encoding == OBJ_ENCODING_LISTPACK) { @@ -720,9 +745,7 @@ void dismissZsetObject(robj *o, size_t size_hint) { } /* Dismiss hash table memory. */ - dict *d = zs->dict; - dismissMemory(d->ht_table[0], DICTHT_SIZE(d->ht_size_exp[0])*sizeof(dictEntry*)); - dismissMemory(d->ht_table[1], DICTHT_SIZE(d->ht_size_exp[1])*sizeof(dictEntry*)); + dismissDictBucketsMemory(zs->dict); } else if (o->encoding == OBJ_ENCODING_LISTPACK) { dismissMemory(o->ptr, lpBytes((unsigned char*)o->ptr)); } else { @@ -748,8 +771,7 @@ void dismissHashObject(robj *o, size_t size_hint) { } /* Dismiss hash table memory. */ - dismissMemory(d->ht_table[0], DICTHT_SIZE(d->ht_size_exp[0])*sizeof(dictEntry*)); - dismissMemory(d->ht_table[1], DICTHT_SIZE(d->ht_size_exp[1])*sizeof(dictEntry*)); + dismissDictBucketsMemory(d); } else if (o->encoding == OBJ_ENCODING_LISTPACK) { dismissMemory(o->ptr, lpBytes((unsigned char*)o->ptr)); } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) { @@ -780,6 +802,13 @@ void dismissStreamObject(robj *o, size_t size_hint) { } } +void dismissGCRAObject(robj *o, size_t size_hint) { + /* GCRA is a single allocation of a long long thus way smaller than a + * page-size. The dismiss mechanism is not needed for it - hence NOOP.*/ + (void)o; + (void)size_hint; +} + /* When creating a snapshot in a fork child process, the main process and child * process share the same physical memory pages, and if / when the parent * modifies any keys due to write traffic, it'll cause CoW which consume @@ -808,6 +837,7 @@ void dismissObject(robj *o, size_t size_hint) { case OBJ_ZSET: dismissZsetObject(o, size_hint); break; case OBJ_HASH: dismissHashObject(o, size_hint); break; case OBJ_STREAM: dismissStreamObject(o, size_hint); break; + case OBJ_GCRA: dismissGCRAObject(o, size_hint); break; default: break; } #else @@ -929,6 +959,7 @@ size_t getObjectLength(robj *o) { case OBJ_ZSET: return zsetLength(o); case OBJ_HASH: return hashTypeLength(o, 0); case OBJ_STREAM: return streamLength(o); + case OBJ_GCRA: return gcraObjectLength(o); default: return 0; } } @@ -1137,6 +1168,22 @@ int getLongLongFromObject(robj *o, long long *target) { return C_OK; } +int getLongLongFromGCRAObject(robj *o, long long *target) { + long long res; + serverAssertWithInfo(NULL, o, o->type == OBJ_GCRA); + serverAssert(o->encoding == OBJ_ENCODING_INT); +#if UINTPTR_MAX == 0xffffffff + res = *((long long*)o->ptr); +#else + res = (long long)o->ptr; +#endif + if (unlikely(res < 0)) { + serverPanic("Invalid negative GCRA value"); + } + *target = res; + return C_OK; +} + int getLongLongFromObjectOrReply(client *c, robj *o, long long *target, const char *msg) { long long value; if (getLongLongFromObject(o, &value) != C_OK) { @@ -1227,7 +1274,8 @@ size_t kvobjComputeSize(robj *key, kvobj *o, size_t sample_size, int dbid) { o->type == OBJ_SET || o->type == OBJ_ZSET || o->type == OBJ_HASH || - o->type == OBJ_STREAM) + o->type == OBJ_STREAM || + o->type == OBJ_GCRA) { return kvobjAllocSize(o); } else if (o->type == OBJ_MODULE) { @@ -1253,12 +1301,31 @@ size_t kvobjAllocSize(kvobj *o) { } else if (o->type == OBJ_STREAM) { stream *s = o->ptr; asize += s->alloc_size; + } else if (o->type == OBJ_GCRA) { + asize += gcraTypeAllocSize(o); } else if (o->type == OBJ_MODULE) { /* TODO: Provide moduleGetAllocSize() module API for O(1) allocation size retrieval */ } return asize; } +size_t gcraTypeAllocSize(robj *o) { + (void)o; +#if UINTPTR_MAX == 0xffffffff + return sizeof(long long); +#else + /* Same as string with int encoding there is no allocation as the value is + * cast to void* and stored in o->ptr */ + return 0; +#endif +} + +/* The gcra object is a single long long value */ +size_t gcraObjectLength(robj *o) { + (void)o; + return 1; +} + /* Release data obtained with getMemoryOverheadData(). */ void freeMemoryOverheadData(struct redisMemOverhead *mh) { zfree(mh->db); diff --git a/src/object.h b/src/object.h index 1e761175d..6b2591877 100644 --- a/src/object.h +++ b/src/object.h @@ -5,7 +5,7 @@ * values of different logical types (strings, lists, sets, hashes, sorted sets, * streams, modules, ...). It contains: * - type: one of OBJ_STRING, OBJ_LIST, OBJ_SET, OBJ_ZSET, OBJ_HASH, OBJ_STREAM, - * OBJ_MODULE, ... + * OBJ_GCRA, OBJ_MODULE, ... * - encoding: an implementation detail of how the value is represented in * memory for the given type (see OBJ_ENCODING_* below). For example, * strings may be RAW/EMBSTR/INT, sets may be INTSET or HT, etc. @@ -161,6 +161,7 @@ robj *createHashObject(void); robj *createZsetObject(void); robj *createZsetListpackObject(void); robj *createStreamObject(void); +robj *createGCRAObject(long long value); robj *createModuleObject(struct RedisModuleType *mt, void *value); int getLongFromObjectOrReply(struct client *c, robj *o, long *target, const char *msg); int getPositiveLongFromObjectOrReply(struct client *c, robj *o, long *target, const char *msg); @@ -170,6 +171,7 @@ int getLongLongFromObjectOrReply(struct client *c, robj *o, long long *target, c int getDoubleFromObjectOrReply(struct client *c, robj *o, double *target, const char *msg); int getDoubleFromObject(const robj *o, double *target); int getLongLongFromObject(robj *o, long long *target); +int getLongLongFromGCRAObject(robj *o, long long *target); int getLongDoubleFromObject(robj *o, long double *target); int getLongDoubleFromObjectOrReply(struct client *c, robj *o, long double *target, const char *msg); int getIntFromObjectOrReply(struct client *c, robj *o, int *target, const char *msg); @@ -179,6 +181,8 @@ int collateStringObjects(const robj *a, const robj *b); int equalStringObjects(robj *a, robj *b); void trimStringObjectIfNeeded(robj *o, int trim_small_values); size_t kvobjAllocSize(kvobj *o); +size_t gcraTypeAllocSize(robj *o); +size_t gcraObjectLength(robj *o); int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, long long lru_clock, int lru_multiplier); diff --git a/src/rdb.c b/src/rdb.c index e880d20be..eae234832 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -713,6 +713,8 @@ int rdbSaveObjectType(rio *rdb, robj *o) { serverPanic("Unknown hash encoding"); case OBJ_STREAM: return rdbSaveType(rdb,RDB_TYPE_STREAM_LISTPACKS_5); + case OBJ_GCRA: + return rdbSaveType(rdb,RDB_TYPE_GCRA); case OBJ_MODULE: return rdbSaveType(rdb,RDB_TYPE_MODULE_2); default: @@ -1399,6 +1401,11 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { /* Save the all-time count of duplicate IIDs detected. */ if ((n = rdbSaveLen(rdb,s->iids_duplicates)) == -1) return -1; nwritten += n; + } else if (o->type == OBJ_GCRA) { + long long t; + getLongLongFromGCRAObject(o, &t); + if ((n = rdbSaveLen(rdb,t)) == -1) return -1; + nwritten += n; } else if (o->type == OBJ_MODULE) { /* Save a module-specific value. */ RedisModuleIO io; @@ -1680,6 +1687,10 @@ ssize_t rdbSaveDb(rio *rdb, int dbid, int rdbflags, long *key_counter, unsigned written += res; if ((res = rdbSaveLen(rdb, kvstoreDictSize(db->expires, curr_slot))) < 0) goto werr2; written += res; + /* Dismiss bucket arrays of the previous slot to reduce CoW. + * The final slot is not dismissed since the child exits shortly after. */ + if (server.in_fork_child && last_slot != -1) + dismissDictBucketsMemory(kvstoreGetDict(db->keys, last_slot)); last_slot = curr_slot; } kvobj *kv = dictGetKV(de); @@ -1707,7 +1718,8 @@ ssize_t rdbSaveDb(rio *rdb, int dbid, int rdbflags, long *key_counter, unsigned * OS and possibly avoid or decrease COW. We give the dismiss * mechanism a hint about an estimated size of the object we stored. */ size_t dump_size = rdb->processed_bytes - rdb_bytes_before_key; - if (server.in_fork_child) dismissObject(kv, dump_size); + if (server.in_fork_child && dump_size > server.page_size/2) + dismissObject(kv, dump_size); /* Update child info every 1 second (approximately). * in order to avoid calling mstime() on each iteration, we will @@ -1758,6 +1770,10 @@ int rdbSaveRio(int req, rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) { if (!(req & SLAVE_REQ_RDB_EXCLUDE_DATA)) { for (j = 0; j < server.dbnum; j++) { if (rdbSaveDb(rdb, j, rdbflags, &key_counter, &skipped) == -1) goto werr; + /* In standalone mode, dismiss bucket arrays of the saved DB's + * kvstore to reduce CoW. In cluster mode this is done per-slot. */ + if (server.in_fork_child && !server.cluster_enabled) + dismissKvstoreBucketsMemory(server.db[j].keys); } } @@ -3595,6 +3611,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) return NULL; } o = createModuleObject(mt, ptr); + } else if (rdbtype == RDB_TYPE_GCRA) { + uint64_t time = rdbLoadLen(rdb, NULL); + if (time == RDB_LENERR || time > LLONG_MAX) { + rdbReportReadError("Failed loading GCRA TaT value"); + return NULL; + } + o = createGCRAObject((long long)time); } else { rdbReportReadError("Unknown RDB encoding type %d",rdbtype); return NULL; diff --git a/src/rdb.h b/src/rdb.h index 5d92f8430..4898d82af 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -80,10 +80,11 @@ #define RDB_TYPE_HASH_LISTPACK_EX 25 /* Hash LP with HFEs. Attach min TTL at start */ #define RDB_TYPE_STREAM_LISTPACKS_4 26 /* Stream with IDMP support */ #define RDB_TYPE_STREAM_LISTPACKS_5 27 /* Stream with XNACK support (NACKed entries) */ +#define RDB_TYPE_GCRA 28 /* GCRA object */ /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType(), and rdb_type_string[] */ /* Test if a type is an object type. */ -#define rdbIsObjectType(t) (((t) >= 0 && (t) <= 7) || ((t) >= 9 && (t) <= 27)) +#define rdbIsObjectType(t) (((t) >= 0 && (t) <= 7) || ((t) >= 9 && (t) <= 28)) /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ #define RDB_OPCODE_KEY_META 243 /* Key metadata (module metadata classes). */ diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c index 4fe226474..eea78290d 100644 --- a/src/redis-check-rdb.c +++ b/src/redis-check-rdb.c @@ -88,6 +88,7 @@ char *rdb_type_string[] = { "hash-listpack-md", "stream-v4", "stream-v5", + "gcra", }; /* Show a few stats collected into 'rdbstate' */ diff --git a/src/redismodule.h b/src/redismodule.h index 71579d3c3..c1040f12f 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -89,6 +89,7 @@ typedef long long ustime_t; #define REDISMODULE_KEYTYPE_ZSET 5 #define REDISMODULE_KEYTYPE_MODULE 6 #define REDISMODULE_KEYTYPE_STREAM 7 +#define REDISMODULE_KEYTYPE_GCRA 8 /* Reply types. */ #define REDISMODULE_REPLY_UNKNOWN -1 @@ -247,11 +248,12 @@ This flag should not be used directly by the module. #define REDISMODULE_NOTIFY_OVERWRITTEN (1<<15) /* o, key overwrite notification */ #define REDISMODULE_NOTIFY_TYPE_CHANGED (1<<16) /* c, key type changed notification */ #define REDISMODULE_NOTIFY_KEY_TRIMMED (1<<17) /* module only key space notification, indicates a key trimmed during slot migration */ +#define REDISMODULE_NOTIFY_RATE_LIMIT (1<<18) /* r, rate limit event */ /* Next notification flag, must be updated when adding new flags above! This flag should not be used directly by the module. * Use RedisModule_GetKeyspaceNotificationFlagsAll instead. */ -#define _REDISMODULE_NOTIFY_NEXT (1<<18) +#define _REDISMODULE_NOTIFY_NEXT (1<<19) #define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM | REDISMODULE_NOTIFY_MODULE) /* A */ diff --git a/src/replication.c b/src/replication.c index f27ab8b7d..2ad39ab6f 100644 --- a/src/replication.c +++ b/src/replication.c @@ -376,23 +376,6 @@ int prepareReplicasToWrite(void) { return prepared; } -/* Wrapper for feedReplicationBuffer() that takes Redis string objects - * as input. */ -void feedReplicationBufferWithObject(robj *o) { - char llstr[LONG_STR_SIZE]; - void *p; - size_t len; - - if (o->encoding == OBJ_ENCODING_INT) { - len = ll2string(llstr,sizeof(llstr),(long)o->ptr); - p = llstr; - } else { - len = sdslen(o->ptr); - p = o->ptr; - } - feedReplicationBuffer(p,len); -} - /* Generally, we only have one replication buffer block to trim when replication * backlog size exceeds our setting and no replica reference it. But if replica * clients disconnect, we need to free many replication buffer blocks that are @@ -468,115 +451,175 @@ void freeReplicaReferencedReplBuffer(client *replica) { replica->ref_block_pos = 0; } -/* Append bytes into the global replication buffer list, replication backlog and - * all replica clients use replication buffers collectively, this function replace - * 'addReply*', 'feedReplicationBacklog' for replicas and replication backlog, - * First we add buffer into global replication buffer block list, and then - * update replica / replication-backlog referenced node and block position. */ -void feedReplicationBuffer(char *s, size_t len) { +/* Batched write API for the global replication backlog, optimized for minimal + * overhead per append: data writes are just memcpys into the tail block. + * All bookkeeping is deferred to replBufWriterEnd(). */ +typedef struct replBufWriter { + listNode *start_node; /* First repl buffer block written to. */ + size_t start_pos; /* Byte offset within start_node where writing began. */ + size_t total_len; /* Total bytes written across all writes. */ + int new_blocks; /* Number of new blocks allocated during this stream. */ + replBufBlock *tail; /* Current tail block. */ +} replBufWriter; + +/* Initialize the writer, cache the current tail position. */ +static void replBufWriterBegin(replBufWriter *wr) { + listNode *ln = listLast(server.repl_buffer_blocks); + replBufBlock *tail = ln ? listNodeValue(ln) : NULL; + + if (tail && tail->used < tail->size) { + wr->start_node = ln; + wr->start_pos = tail->used; + } else { + wr->start_node = NULL; + wr->start_pos = 0; + } + + wr->total_len = 0; + wr->new_blocks = 0; + wr->tail = tail; +} + +/* Allocate a new replication backlog block. Called when current block is full. */ +static void replBufWriterAllocBlock(replBufWriter *wr, size_t hint) { static long long repl_block_id = 0; + size_t usable_size; + /* Avoid creating nodes smaller than PROTO_REPLY_CHUNK_BYTES, so that we can append more data into them, + * and also avoid creating nodes bigger than repl_backlog_size / 16, so that we won't have huge nodes that can't + * trim when we only still need to hold a small portion from them. */ + size_t limit = max((size_t)server.repl_backlog_size / 16, (size_t)PROTO_REPLY_CHUNK_BYTES); + size_t bsize = min(max(hint, (size_t)PROTO_REPLY_CHUNK_BYTES), limit); + replBufBlock *tail = zmalloc_usable(bsize + sizeof(replBufBlock), &usable_size); + /* Take over the allocation's internal fragmentation */ + tail->size = usable_size - sizeof(replBufBlock); + tail->used = 0; + tail->refcount = 0; + tail->repl_offset = server.master_repl_offset + wr->total_len + 1; + tail->id = repl_block_id++; + listAddNodeTail(server.repl_buffer_blocks, tail); + server.repl_buffer_mem += (usable_size + sizeof(listNode)); + createReplicationBacklogIndex(listLast(server.repl_buffer_blocks)); - if (server.repl_backlog == NULL) return; + /* Update stream state. */ + wr->tail = tail; + wr->new_blocks++; + if (wr->start_node == NULL) { + wr->start_node = listLast(server.repl_buffer_blocks); + wr->start_pos = 0; + } +} - clusterSlotStatsIncrNetworkBytesOutForReplication(len); +/* Slow path: fill remainder of current block + allocate as needed. */ +static void replBufWriterAppendSlow(replBufWriter *wr, const char *buf, size_t len) { + while (len > 0) { + size_t avail = wr->tail ? wr->tail->size - wr->tail->used : 0; + if (avail > 0) { + size_t copy = (avail >= len) ? len : avail; + memcpy(wr->tail->buf + wr->tail->used, buf, copy); + wr->tail->used += copy; + wr->total_len += copy; + buf += copy; + len -= copy; + } + + if (len > 0) + replBufWriterAllocBlock(wr, len); + } +} + +/* Write data into the replication buffer. The slow path is split out to give + * the compiler a chance to inline the common case where the write fits entirely + * in the current block. */ +static inline void replBufWriterAppend(replBufWriter *wr, const char *buf, size_t len) { + size_t avail = wr->tail ? wr->tail->size - wr->tail->used : 0; + if (len > 0 && avail >= len) { + memcpy(wr->tail->buf + wr->tail->used, buf, len); + wr->tail->used += len; + wr->total_len += len; + return; + } + replBufWriterAppendSlow(wr, buf, len); +} + +/* Write a RESP header prefix\r\n (e.g. "$12\r\n" or "*3\r\n"). + * Uses pre-built shared objects for small values, formats manually otherwise. */ +static inline void replBufWriterAppendBulkLen(replBufWriter *wr, char prefix, long long value) { + serverAssert(prefix == '$' || prefix == '*'); + if (value >= 0 && value < OBJ_SHARED_BULKHDR_LEN) { + robj **tbl = (prefix == '$') ? shared.bulkhdr : shared.mbulkhdr; + replBufWriterAppend(wr, tbl[value]->ptr, OBJ_SHARED_HDR_STRLEN(value)); + return; + } + char buf[LONG_STR_SIZE+3]; + buf[0] = prefix; + int len = ll2string(buf+1, sizeof(buf)-1, value); + buf[len+1] = '\r'; + buf[len+2] = '\n'; + replBufWriterAppend(wr, buf, len+3); +} + + +/* Finalize the replication buffer write: update global offsets, set up replica + * references for new data, check output buffer limits, and trim the + * backlog if new blocks were allocated. */ +static void replBufWriterEnd(replBufWriter *wr) { + if (wr->total_len == 0) return; + + serverAssert(wr->start_node != NULL); + clusterSlotStatsIncrNetworkBytesOutForReplication(wr->total_len); /* Update the current cmd's keys with the commands replication bytes*/ - hotkeyMetrics metrics = {0, len}; + hotkeyMetrics metrics = {0, wr->total_len}; hotkeyStatsUpdateCurrentCmd(server.hotkeys, metrics); - while(len > 0) { - size_t start_pos = 0; /* The position of referenced block to start sending. */ - listNode *start_node = NULL; /* Replica/backlog starts referenced node. */ - int add_new_block = 0; /* Create new block if current block is total used. */ - listNode *ln = listLast(server.repl_buffer_blocks); - replBufBlock *tail = ln ? listNodeValue(ln) : NULL; + server.master_repl_offset += wr->total_len; + server.repl_backlog->histlen += wr->total_len; - /* Append to tail string when possible. */ - if (tail && tail->size > tail->used) { - start_node = listLast(server.repl_buffer_blocks); - start_pos = tail->used; - /* Copy the part we can fit into the tail, and leave the rest for a - * new node */ - size_t avail = tail->size - tail->used; - size_t copy = (avail >= len) ? len : avail; - memcpy(tail->buf + tail->used, s, copy); - tail->used += copy; - s += copy; - len -= copy; - server.master_repl_offset += copy; - server.repl_backlog->histlen += copy; - } - if (len) { - /* Create a new node, make sure it is allocated to at - * least PROTO_REPLY_CHUNK_BYTES */ - size_t usable_size; - /* Avoid creating nodes smaller than PROTO_REPLY_CHUNK_BYTES, so that we can append more data into them, - * and also avoid creating nodes bigger than repl_backlog_size / 16, so that we won't have huge nodes that can't - * trim when we only still need to hold a small portion from them. */ - size_t limit = max((size_t)server.repl_backlog_size / 16, (size_t)PROTO_REPLY_CHUNK_BYTES); - size_t size = min(max(len, (size_t)PROTO_REPLY_CHUNK_BYTES), limit); - tail = zmalloc_usable(size + sizeof(replBufBlock), &usable_size); - /* Take over the allocation's internal fragmentation */ - tail->size = usable_size - sizeof(replBufBlock); - size_t copy = (tail->size >= len) ? len : tail->size; - tail->used = copy; - tail->refcount = 0; - tail->repl_offset = server.master_repl_offset + 1; - tail->id = repl_block_id++; - memcpy(tail->buf, s, copy); - listAddNodeTail(server.repl_buffer_blocks, tail); - /* We also count the list node memory into replication buffer memory. */ - server.repl_buffer_mem += (usable_size + sizeof(listNode)); - add_new_block = 1; - if (start_node == NULL) { - start_node = listLast(server.repl_buffer_blocks); - start_pos = 0; - } - s += copy; - len -= copy; - server.master_repl_offset += copy; - server.repl_backlog->histlen += copy; - } + /* For output buffer of replicas. */ + listIter li; + listNode *ln; + listRewind(server.slaves,&li); + while((ln = listNext(&li))) { + client *slave = ln->value; + if (!canFeedReplicaReplBuffer(slave)) continue; - /* For output buffer of replicas. */ - listIter li; - listRewind(server.slaves,&li); - while((ln = listNext(&li))) { - client *slave = ln->value; - if (!canFeedReplicaReplBuffer(slave)) continue; - - /* Update shared replication buffer start position. */ - if (slave->ref_repl_buf_node == NULL) { - slave->ref_repl_buf_node = start_node; - slave->ref_block_pos = start_pos; - /* Only increase the start block reference count. */ - ((replBufBlock *)listNodeValue(start_node))->refcount++; - } - - /* Check output buffer limit only when add new block. */ - if (add_new_block) closeClientOnOutputBufferLimitReached(slave, 1); - } - - /* For replication backlog */ - if (server.repl_backlog->ref_repl_buf_node == NULL) { - server.repl_backlog->ref_repl_buf_node = start_node; + /* Update shared replication buffer start position. */ + if (slave->ref_repl_buf_node == NULL) { + slave->ref_repl_buf_node = wr->start_node; + slave->ref_block_pos = wr->start_pos; /* Only increase the start block reference count. */ - ((replBufBlock *)listNodeValue(start_node))->refcount++; - - /* Replication buffer must be empty before adding replication stream - * into replication backlog. */ - serverAssert(add_new_block == 1 && start_pos == 0); + ((replBufBlock *)listNodeValue(wr->start_node))->refcount++; } - if (add_new_block) { - createReplicationBacklogIndex(listLast(server.repl_buffer_blocks)); - /* It is important to trim after adding replication data to keep the backlog size close to - * repl_backlog_size in the common case. We wait until we add a new block to avoid repeated - * unnecessary trimming attempts when small amounts of data are added. See comments in - * freeMemoryGetNotCountedMemory() for details on replication backlog memory tracking. */ - incrementalTrimReplicationBacklog(REPL_BACKLOG_TRIM_BLOCKS_PER_CALL); - } + /* Check output buffer limit only when new blocks were added. */ + if (wr->new_blocks) closeClientOnOutputBufferLimitReached(slave, 1); } + + /* For replication backlog */ + if (server.repl_backlog->ref_repl_buf_node == NULL) { + server.repl_backlog->ref_repl_buf_node = wr->start_node; + /* Only increase the start block reference count. */ + ((replBufBlock *)listNodeValue(wr->start_node))->refcount++; + + /* Replication buffer must be empty before adding replication stream + * into replication backlog. */ + serverAssert(wr->new_blocks > 0 && wr->start_pos == 0); + } + if (wr->new_blocks) { + /* It is important to trim after adding replication data to keep the backlog size close to + * repl_backlog_size in the common case. We wait until we add a new block to avoid repeated + * unnecessary trimming attempts when small amounts of data are added. See comments in + * freeMemoryGetNotCountedMemory() for details on replication backlog memory tracking. */ + incrementalTrimReplicationBacklog(REPL_BACKLOG_TRIM_BLOCKS_PER_CALL); + } +} + +/* Append bytes into the global replication buffer. */ +static void feedReplicationBuffer(const char *buf, size_t len) { + replBufWriter wr; + replBufWriterBegin(&wr); + replBufWriterAppend(&wr, buf, len); + replBufWriterEnd(&wr); } /* Propagate write commands to replication stream. @@ -642,7 +685,7 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { dictid_len, llstr)); } - feedReplicationBufferWithObject(selectcmd); + feedReplicationBuffer(selectcmd->ptr, sdslen(selectcmd->ptr)); /* Although the SELECT command is not associated with any slot, * its per-slot network-bytes-out accumulation is made by the above function call. @@ -657,28 +700,28 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { /* Write the command to the replication buffer if any. */ char aux[LONG_STR_SIZE+3]; + replBufWriter wr; + replBufWriterBegin(&wr); - /* Add the multi bulk reply length. */ - aux[0] = '*'; - len = ll2string(aux+1,sizeof(aux)-1,argc); - aux[len+1] = '\r'; - aux[len+2] = '\n'; - feedReplicationBuffer(aux,len+3); + /* Write the multi bulk count */ + replBufWriterAppendBulkLen(&wr, '*', argc); for (j = 0; j < argc; j++) { + /* Write the bulk count */ long objlen = stringObjectLen(argv[j]); + replBufWriterAppendBulkLen(&wr, '$', objlen); - /* We need to feed the buffer with the object as a bulk reply - * not just as a plain string, so create the $..CRLF payload len - * and add the final CRLF */ - aux[0] = '$'; - len = ll2string(aux+1,sizeof(aux)-1,objlen); - aux[len+1] = '\r'; - aux[len+2] = '\n'; - feedReplicationBuffer(aux,len+3); - feedReplicationBufferWithObject(argv[j]); - feedReplicationBuffer(aux+len+1,2); + /* Write the bulk data */ + if (argv[j]->encoding == OBJ_ENCODING_INT) { + len = ll2string(aux, sizeof(aux), (long)argv[j]->ptr); + replBufWriterAppend(&wr, aux, len); + } else { + replBufWriterAppend(&wr, argv[j]->ptr, objlen); + } + replBufWriterAppend(&wr, "\r\n", 2); } + + replBufWriterEnd(&wr); } /* This is a debugging function that gets called when we detect something diff --git a/src/resp_parser.c b/src/resp_parser.c index 8c0f17d39..fd1b5acc1 100644 --- a/src/resp_parser.c +++ b/src/resp_parser.c @@ -128,13 +128,10 @@ static int parseDouble(ReplyParser *parser, void *p_ctx) { const char *proto = parser->curr_location; char *p = strchr(proto+1,'\r'); parser->curr_location = p + 2; /* for \r\n */ - char buf[MAX_LONG_DOUBLE_CHARS+1]; size_t len = p-proto-1; double d; if (len <= MAX_LONG_DOUBLE_CHARS) { - memcpy(buf,proto+1,len); - buf[len] = '\0'; - d = fast_float_strtod(buf,NULL); /* We expect a valid representation. */ + d = fast_float_strtod(proto+1,len,NULL); /* We expect a valid representation. */ } else { d = 0; } diff --git a/src/server.c b/src/server.c index ffc5d1f7c..40617ce80 100644 --- a/src/server.c +++ b/src/server.c @@ -32,6 +32,7 @@ #include "fwtree.h" #include "estore.h" #include "chk.h" +#include "fast_float_strtod.h" #include #include @@ -581,6 +582,19 @@ dictType objectKeyPointerValueDictType = { NULL /* allow to expand */ }; +/* Dict type with robj pointer keys and no values. */ +dictType objectKeyNoValueDictType = { + dictEncObjHash, /* hash function */ + NULL, /* key dup */ + NULL, /* val dup */ + dictEncObjKeyCompare, /* key compare */ + dictObjectDestructor, /* key destructor */ + NULL, /* val destructor */ + NULL, /* allow to expand */ + .no_value = 1, /* no values in this dict */ + .keys_are_odd = 0, /* robj pointers are not odd */ +}; + /* Like objectKeyPointerValueDictType(), but values can be destroyed, if * not NULL, calling zfree(). */ dictType objectKeyHeapPointerValueDictType = { @@ -2996,7 +3010,7 @@ void initServer(void) { server.db[j].blocking_keys = dictCreate(&keylistDictType); server.db[j].blocking_keys_unblock_on_nokey = dictCreate(&objectKeyPointerValueDictType); server.db[j].stream_claim_pending_keys = dictCreate(&objectKeyPointerValueDictType); - server.db[j].stream_idmp_keys = dictCreate(&objectKeyPointerValueDictType); + server.db[j].stream_idmp_keys = dictCreate(&objectKeyNoValueDictType); server.db[j].ready_keys = dictCreate(&objectKeyPointerValueDictType); server.db[j].watched_keys = dictCreate(&keylistDictType); server.db[j].id = j; @@ -7483,6 +7497,20 @@ void dismissClientMemory(client *c) { } } +/* Dismiss the hash table bucket arrays of a dict. */ +void dismissDictBucketsMemory(dict *d) { + if (!d) return; + dismissMemory(d->ht_table[0], DICTHT_SIZE(d->ht_size_exp[0]) * sizeof(dictEntry*)); + dismissMemory(d->ht_table[1], DICTHT_SIZE(d->ht_size_exp[1]) * sizeof(dictEntry*)); +} + +/* Dismiss the hash table bucket arrays for all dicts in the given kvstore. */ +void dismissKvstoreBucketsMemory(kvstore *kvs) { + for (int didx = 0; didx < kvstoreNumDicts(kvs); didx++) { + dismissDictBucketsMemory(kvstoreGetDict(kvs, didx)); + } +} + /* In the child process, we don't need some buffers anymore, and these are * likely to change in the parent when there's heavy write traffic. * We dismiss them right away, to avoid CoW. @@ -7521,6 +7549,14 @@ void dismissMemoryInChild(void) { client *c = listNodeValue(ln); dismissClientMemory(c); } + + /* Dismiss expires kvstore bucket arrays since the child process never + * accesses them, expire times are embedded in key objects. */ + if (server.in_fork_child == CHILD_TYPE_RDB || server.in_fork_child == CHILD_TYPE_AOF) { + for (int dbid = 0; dbid < server.dbnum; dbid++) { + dismissKvstoreBucketsMemory(server.db[dbid].expires); + } + } #endif } @@ -7784,6 +7820,7 @@ int __test_num = 0; typedef int redisTestProc(int argc, char **argv, int flags); int bitopsTest(int argc, char **argv, int flags); int zsetTest(int argc, char **argv, int flags); +int vectorTest(int argc, char **argv, int flags); struct redisTest { char *name; redisTestProc *proc; @@ -7807,11 +7844,13 @@ struct redisTest { {"fwtree", fwtreeTest}, {"estore", estoreTest}, {"ebuckets", ebucketsTest}, + {"vector", vectorTest}, {"bitmap", bitopsTest}, {"rax", raxTest}, {"flax", flaxTest}, {"zset", zsetTest}, {"topk", chkTopKTest}, + {"fastfloat", fastFloatTest}, }; redisTestProc *getTestProcByName(const char *name) { int numtests = sizeof(redisTests)/sizeof(struct redisTest); diff --git a/src/server.h b/src/server.h index dbc9a6593..cc691abb0 100644 --- a/src/server.h +++ b/src/server.h @@ -288,6 +288,7 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; #define ACL_CATEGORY_CONNECTION (1ULL<<18) #define ACL_CATEGORY_TRANSACTION (1ULL<<19) #define ACL_CATEGORY_SCRIPTING (1ULL<<20) +#define ACL_CATEGORY_RATE_LIMIT (1ULL<<21) /* Key-spec flags * * -------------- */ @@ -796,6 +797,7 @@ typedef enum { #define NOTIFY_OVERWRITTEN (1<<15) /* o, key overwrite notification (Note: excluded from NOTIFY_ALL) */ #define NOTIFY_TYPE_CHANGED (1<<16) /* c, key type changed notification (Note: excluded from NOTIFY_ALL) */ #define NOTIFY_KEY_TRIMMED (1<<17) /* module only key space notification, indicates a key trimmed during slot migration */ +#define NOTIFY_RATE_LIMIT (1<<18) /* r, notify rate limit event (Note: excluded from NOTIFY_ALL)*/ #define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM | NOTIFY_MODULE) /* A flag */ /* Using the following macro you can run code inside serverCron() with the @@ -860,7 +862,17 @@ typedef enum { * encoding version. */ #define OBJ_MODULE 5 /* Module object. */ #define OBJ_STREAM 6 /* Stream object. */ -#define OBJ_TYPE_MAX 7 /* Maximum number of object types */ +#define OBJ_GCRA 7 /* GCRA object. */ +#define OBJ_TYPE_MAX 8 /* Maximum number of object types */ + +/* NOTE: adding a new object requires changes in the following places: + * - rdb.c - save/load (also bump RDB_VERSION if needed) + * - aof.c - rewrite + * - db.c - obj_type_name, copyCommand + * - debug.c - xorObjectDigest, serverLogObjectDebugInfo + * - defrag.c - defragKey + * - module.c - RM_KeyType (and add the new keytype to redismodule.h) + * - object.c - object(create/free/dismiss/allocSize/Length) */ /* Extract encver / signature from a module type ID. */ #define REDISMODULE_TYPE_ENCVER_BITS 10 @@ -2772,6 +2784,7 @@ typedef enum { COMMAND_GROUP_STREAM, COMMAND_GROUP_BITMAP, COMMAND_GROUP_MODULE, + COMMAND_GROUP_RATE_LIMIT, } redisCommandGroup; typedef void redisCommandProc(client *c); @@ -3005,6 +3018,7 @@ typedef struct { extern struct redisServer server; extern struct sharedObjectsStruct shared; extern dictType objectKeyPointerValueDictType; +extern dictType objectKeyNoValueDictType; extern dictType objectKeyHeapPointerValueDictType; extern dictType setDictType; extern dictType BenchmarkDictType; @@ -3360,7 +3374,6 @@ ssize_t syncReadLine(int fd, char *ptr, ssize_t size, long long timeout); void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc); void replicationFeedStreamFromMasterStream(char *buf, size_t buflen); void resetReplicationBuffer(void); -void feedReplicationBuffer(char *buf, size_t len); void freeReplicaReferencedReplBuffer(client *replica); void replicationFeedMonitors(client *c, list *monitors, int dictid, robj **argv, int argc); void updateSlavesWaitingBgsave(int bgsaveerr, int type); @@ -3598,6 +3611,9 @@ int zzlLexValueLteMax(unsigned char *p, zlexrangespec *spec); int zslLexValueGteMin(sds value, zlexrangespec *spec); int zslLexValueLteMax(sds value, zlexrangespec *spec); +/* gcra related */ +robj *gcraDup(robj *o); + /* Core functions */ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *level); void updatePeakMemory(void); @@ -3682,6 +3698,8 @@ void activeDefragFreeRaw(void *ptr); robj *activeDefragStringOb(robj* ob); void dismissSds(sds s); void dismissMemory(void* ptr, size_t size_hint); +void dismissDictBucketsMemory(dict *d); +void dismissKvstoreBucketsMemory(kvstore *kvs); void dismissMemoryInChild(void); int clientsCronRunClient(client *c); @@ -4468,6 +4486,7 @@ void resetCommand(client *c); void failoverCommand(client *c); void digestCommand(client *c); void gcraCommand(client *c); +void gcraSetValueCommand(client *c); #if defined(__GNUC__) void *calloc(size_t count, size_t size) __attribute__ ((deprecated)); diff --git a/src/sort.c b/src/sort.c index c6b32624e..0d8dcdd9b 100644 --- a/src/sort.c +++ b/src/sort.c @@ -518,12 +518,7 @@ void sortCommandGeneric(client *c, int readonly) { if (sortby) vector[j].u.cmpobj = getDecodedObject(byval); } else { if (sdsEncodedObject(byval)) { - char *eptr; - - vector[j].u.score = fast_float_strtod(byval->ptr,&eptr); - if (eptr[0] != '\0' || errno == ERANGE || - isnan(vector[j].u.score)) - { + if (string2d(byval->ptr,sdslen(byval->ptr),&vector[j].u.score) == 0) { int_conversion_error = 1; } } else if (byval->encoding == OBJ_ENCODING_INT) { diff --git a/src/t_hash.c b/src/t_hash.c index acfa6c6a9..5ea456597 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -1022,7 +1022,6 @@ int hashTypeSet(redisDb *db, kvobj *o, sds field, sds value, int flags) { if (newExpireAt != EB_EXPIRE_TIME_INVALID) { dict *d = o->ptr; htMetadataEx *dictExpireMeta = htGetMetadataEx(d); - serverAssert(dictExpireMeta->expireMeta.trash == 0); ebAdd(&dictExpireMeta->hfe, &hashFieldExpireBucketsType, newEntry, newExpireAt); } @@ -3478,9 +3477,6 @@ static void hfieldPersist(robj *hashObj, Entry *entry) { dict *d = hashObj->ptr; htMetadataEx *dictExpireMeta = htGetMetadataEx(d); - /* If field has valid expiry then dict must have valid metadata as well */ - serverAssert(dictExpireMeta->expireMeta.trash == 0); - /* Remove field from private HFE DS */ ebRemove(&dictExpireMeta->hfe, &hashFieldExpireBucketsType, entry); @@ -3608,15 +3604,16 @@ static int parseHashCommandArgs(client *c, HashCommandArgs *args, &numFields, "Parameter `numFields` should be greater than 0") != C_OK) return C_ERR; - args->fieldCount = (int)numFields; args->firstFieldPos = i + 2; /* Check bounds - we must have exactly the right number of fields */ - if (args->firstFieldPos + args->fieldCount > c->argc) { + if (numFields > c->argc - args->firstFieldPos) { addReplyError(c, "wrong number of arguments"); return C_ERR; } + args->fieldCount = (int)numFields; + /* Skip over the field arguments */ i = args->firstFieldPos + args->fieldCount - 1; continue; diff --git a/src/t_stream.c b/src/t_stream.c index bf4422a7c..7e8f58abd 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -2500,11 +2500,14 @@ void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupna decrRefCount(argv[6]); } -/* We need this when we want to propagate creation of consumer that was created - * by XREADGROUP with the NOACK option. In that case, the only way to create - * the consumer at the replica is by using XGROUP CREATECONSUMER (see issue #7140) +/* Propagate creation of a consumer that was implicitly created by XREADGROUP. + * Called only when no XCLAIM commands were propagated for this consumer, + * since XCLAIM implicitly creates the consumer on the replica. This covers + * two cases: + * (1) NOACK, where the PEL/XCLAIM path is skipped entirely. + * (2) no messages were available to deliver (see #7140). * - * XGROUP CREATECONSUMER + * XGROUP CREATECONSUMER */ void streamPropagateConsumerCreation(client *c, robj *key, robj *groupname, sds consumername) { robj *argv[5]; @@ -3456,6 +3459,7 @@ void xreadCommand(client *c) { int serve_claimed = 0; int serve_synchronously = 0; int serve_history = 0; /* True for XREADGROUP with ID != ">". */ + int consumer_created = 0; streamConsumer *consumer = NULL; /* Unused if XREAD */ streamPropInfo spi = {c->argv[streams_arg+i],groupname}; /* Unused if XREAD */ @@ -3516,10 +3520,7 @@ void xreadCommand(client *c) { c->db->id,SCC_DEFAULT); if (server.memory_tracking_enabled) updateSlotAllocSize(c->db,getKeySlot(c->argv[streams_arg+i]->ptr),o,old_alloc,kvobjAllocSize(o)); - if (noack) - streamPropagateConsumerCreation(c,spi.keyname, - spi.groupname, - consumer->name); + consumer_created = 1; } consumer->seen_time = commandTimeSnapshot(); keyModified(c,c->db,c->argv[streams_arg+i],o,0); /* only update LRM */ @@ -3545,6 +3546,7 @@ void xreadCommand(client *c) { flags |= STREAM_RWR_CLAIMED; } + unsigned long propCount = 0; if (serve_synchronously) { arraylen++; if (arraylen == 1) arraylen_ptr = addReplyDeferredLen(c); @@ -3559,7 +3561,6 @@ void xreadCommand(client *c) { if (c->resp == 2) addReplyArrayLen(c,2); addReplyBulk(c,c->argv[streams_arg+i]); - unsigned long propCount = 0; if (noack) flags |= STREAM_RWR_NOACK; if (serve_history) flags |= STREAM_RWR_HISTORY; if (server.memory_tracking_enabled) @@ -3574,6 +3575,14 @@ void xreadCommand(client *c) { keyModified(c,c->db,c->argv[streams_arg+i],o,0); /* only update LRM */ } } + + /* Propagate consumer creation only when no XCLAIM was generated, + * since XCLAIM implicitly creates the consumer on the replica. + * With NOACK the PEL/XCLAIM path is skipped entirely, so we + * always need explicit propagation regardless of propCount. */ + if (consumer_created && (noack || propCount == 0)) { + streamPropagateConsumerCreation(c,spi.keyname, spi.groupname, consumer->name); + } } /* We replied synchronously! Set the top array len and return to caller. */ @@ -5546,7 +5555,7 @@ void xtrimCommand(client *c) { /* Helper function for xinfoCommand. * Handles the variants of XINFO STREAM */ -void xinfoReplyWithStreamInfo(client *c, kvobj *kv) { +void xinfoReplyWithStreamInfo(client *c, robj *key, kvobj *kv) { stream *s = kv->ptr; int full = 1; long long count = 10; /* Default COUNT is 10 so we don't block the server */ @@ -5779,7 +5788,7 @@ void xinfoReplyWithStreamInfo(client *c, kvobj *kv) { } } if (server.memory_tracking_enabled) - updateSlotAllocSize(c->db,getKeySlot(c->argv[1]->ptr),kv,old_alloc,kvobjAllocSize(kv)); + updateSlotAllocSize(c->db,getKeySlot(key->ptr),kv,old_alloc,kvobjAllocSize(kv)); } /* XINFO CONSUMERS @@ -5883,7 +5892,7 @@ NULL raxStop(&ri); } else if (!strcasecmp(opt,"STREAM")) { /* XINFO STREAM [FULL [COUNT ]]. */ - xinfoReplyWithStreamInfo(c,kv); + xinfoReplyWithStreamInfo(c,key,kv); } else { addReplySubcommandSyntaxError(c); } @@ -6487,7 +6496,7 @@ void streamKeyLoaded(redisDb *db, robj *key, robj *val) { } } -/* To be used when a steam key was removed from ram, un-redigster from stream_idmp_keys if needed */ +/* To be used when a stream key was removed from ram, un-register from stream_idmp_keys if needed */ void streamKeyRemoved(redisDb *db, robj *key, robj *val) { UNUSED(val); dictDelete(db->stream_idmp_keys, key); diff --git a/src/t_zset.c b/src/t_zset.c index 346bcd38c..ff61afdd3 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -721,24 +721,26 @@ static int zslParseRange(robj *min, robj *max, zrangespec *spec) { if (min->encoding == OBJ_ENCODING_INT) { spec->min = (long)min->ptr; } else { + size_t len = sdslen(min->ptr); if (((char*)min->ptr)[0] == '(') { - spec->min = fast_float_strtod((char*)min->ptr+1,&eptr); + spec->min = fast_float_strtod((char*)min->ptr+1,len-1,&eptr); if (eptr[0] != '\0' || isnan(spec->min)) return C_ERR; spec->minex = 1; } else { - spec->min = fast_float_strtod((char*)min->ptr,&eptr); + spec->min = fast_float_strtod((char*)min->ptr,len,&eptr); if (eptr[0] != '\0' || isnan(spec->min)) return C_ERR; } } if (max->encoding == OBJ_ENCODING_INT) { spec->max = (long)max->ptr; } else { + size_t len = sdslen(max->ptr); if (((char*)max->ptr)[0] == '(') { - spec->max = fast_float_strtod((char*)max->ptr+1,&eptr); + spec->max = fast_float_strtod((char*)max->ptr+1,len-1,&eptr); if (eptr[0] != '\0' || isnan(spec->max)) return C_ERR; spec->maxex = 1; } else { - spec->max = fast_float_strtod((char*)max->ptr,&eptr); + spec->max = fast_float_strtod((char*)max->ptr,len,&eptr); if (eptr[0] != '\0' || isnan(spec->max)) return C_ERR; } } @@ -945,13 +947,8 @@ zskiplistNode *zslNthInLexRange(zskiplist *zsl, zlexrangespec *range, long n, un *----------------------------------------------------------------------------*/ static double zzlStrtod(unsigned char *vstr, unsigned int vlen) { - char buf[128]; - if (vlen > sizeof(buf) - 1) - vlen = sizeof(buf) - 1; - memcpy(buf,vstr,vlen); - buf[vlen] = '\0'; - return fast_float_strtod(buf,NULL); - } + return fast_float_strtod((char*)vstr, vlen, NULL); +} double zzlGetScore(unsigned char *sptr) { unsigned char *vstr; @@ -2653,6 +2650,15 @@ static int zuiCompareByRevCardinality(const void *s1, const void *s2) { #define REDIS_AGGR_SUM 1 #define REDIS_AGGR_MIN 2 #define REDIS_AGGR_MAX 3 +#define REDIS_AGGR_COUNT 4 + +/* Return the weighted contribution of a single sorted set member. + * For COUNT aggregation the actual score is irrelevant — each member + * contributes its set's weight (i.e. "one occurrence worth "). + * For all other aggregation modes the contribution is weight * score. */ +inline static double zuiWeightedScore(double score, double weight, int aggregate) { + return (aggregate == REDIS_AGGR_COUNT) ? weight : weight * score; +} inline static void zunionInterAggregate(double *target, double val, int aggregate) { if (aggregate == REDIS_AGGR_SUM) { @@ -2661,6 +2667,11 @@ inline static void zunionInterAggregate(double *target, double val, int aggregat * is +inf and the other is -inf. When these numbers are added, * we maintain the convention of the result being 0.0. */ if (isnan(*target)) *target = 0.0; + } else if (aggregate == REDIS_AGGR_COUNT) { + *target += val; + /* The val is zuiWeightedScore(…) == weight, which can be +inf/-inf, + * so the NaN guard applies here. */ + if (isnan(*target)) *target = 0.0; } else if (aggregate == REDIS_AGGR_MIN) { *target = val < *target ? val : *target; } else if (aggregate == REDIS_AGGR_MAX) { @@ -2962,6 +2973,8 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in aggregate = REDIS_AGGR_MIN; } else if (!strcasecmp(c->argv[j]->ptr,"max")) { aggregate = REDIS_AGGR_MAX; + } else if (!strcasecmp(c->argv[j]->ptr,"count")) { + aggregate = REDIS_AGGR_COUNT; } else { zfree(src); addReplyErrorObject(c,shared.syntaxerr); @@ -3018,17 +3031,17 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in while (zuiNext(&src[0],&zval)) { double score, value; - score = src[0].weight * zval.score; + score = zuiWeightedScore(zval.score, src[0].weight, aggregate); if (isnan(score)) score = 0; for (j = 1; j < setnum; j++) { /* It is not safe to access the zset we are * iterating, so explicitly check for equal object. */ if (src[j].subject == src[0].subject) { - value = zval.score*src[j].weight; + value = zuiWeightedScore(zval.score, src[j].weight, aggregate); zunionInterAggregate(&score,value,aggregate); } else if (zuiFind(&src[j],&zval,&value)) { - value *= src[j].weight; + value = zuiWeightedScore(value, src[j].weight, aggregate); zunionInterAggregate(&score,value,aggregate); } else { break; @@ -3075,7 +3088,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in zuiInitIterator(&src[i]); while (zuiNext(&src[i],&zval)) { /* Initialize value */ - score = src[i].weight * zval.score; + score = zuiWeightedScore(zval.score, src[i].weight, aggregate); if (isnan(score)) score = 0; /* Search for this element in the dict (which stores node pointers). */ diff --git a/src/util.c b/src/util.c index ba3d9d072..becf1486a 100644 --- a/src/util.c +++ b/src/util.c @@ -664,13 +664,10 @@ int string2d(const char *s, size_t slen, double *dp) { if (unlikely(slen == 0 || isspace(((const char*)s)[0]))) return 0; - *dp = fast_float_strtod(s, &eptr); - /* If `fast_float_strtod` didn't consume full input, try `strtod` - * Given fast_float does not support hexadecimal strings representation */ + *dp = fast_float_strtod(s, slen, &eptr); + /* Reject if not all characters were consumed by the parser. */ if (unlikely((size_t)(eptr - (char*)s) != slen)) { - char *fallback_eptr; - *dp = strtod(s, &fallback_eptr); - if ((size_t)(fallback_eptr - (char*)s) != slen) return 0; + return 0; } if (unlikely(errno == EINVAL || (errno == ERANGE && diff --git a/src/vector.c b/src/vector.c new file mode 100644 index 000000000..e5809dabb --- /dev/null +++ b/src/vector.c @@ -0,0 +1,173 @@ +/* vector.c - Simple append-only vector implementation + * + * Copyright (c) 2026-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of (a) the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#include +#include +#include + +#include "vector.h" +#include "redisassert.h" +#include "zmalloc.h" + +#define VEC_DEFAULT_INITCAP 8 + +/* + * Vector initialization. + * + * Modes: + * - stack != NULL: use caller-provided storage for the first initcap items. + * - stack == NULL && initcap > 0: start heap-backed with an initial 'initcap' capacity. + * - stack == NULL && initcap == 0: start heap-backed with no initial storage. + */ +void vecInit(vec *v, void **stack, size_t initcap) { + /* If stack is provided, initcap must be > 0 and at the size of the stack */ + assert(initcap > 0 || stack == NULL); + + v->size = 0; + v->cap = initcap; + v->stack = stack; /* stack is NULL if not used */ + + /* now init data either stack, heap or NULL */ + v->data = (stack) ? stack : ((initcap > 0) ? zmalloc(initcap * sizeof(void *)) : NULL); +} + +/* Free only heap storage if any */ +void vecRelease(vec *v) { + /* if data is not stack-allocated and is not NULL, free it */ + if (v->data && v->data != v->stack) + zfree(v->data); + v->size = 0; + v->cap = 0; + v->data = NULL; + v->stack = NULL; +} + +/* Reset the logical length to zero while preserving allocated storage. */ +void vecClear(vec *v) { + v->size = 0; +} + +/* Return the number of elements in the vector. */ +size_t vecSize(const vec *v) { + return v->size; +} + +/* Get element at index. index must be < vecSize(v). */ +void *vecGet(const vec *v, size_t index) { + assert(index < v->size); + return v->data[index]; +} + +/* Return the contiguous backing array. */ +void **vecData(vec *v) { + return v->data; +} + +/* Ensure capacity is at least mincap. */ +void vecReserve(vec *v, size_t mincap) { + void **newdata; + + if (mincap <= v->cap) return; + + /* If no heap storage is used yet, allocate and copy from stack if needed. */ + if (v->data == v->stack) { + newdata = zmalloc(mincap * sizeof(void *)); + if (v->size) memcpy(newdata, v->data, v->size * sizeof(void *)); + } else { + newdata = zrealloc(v->data, mincap * sizeof(void *)); + } + + v->data = newdata; + v->cap = mincap; +} + +/* Append one element, growing storage as needed. */ +void vecPush(vec *v, void *value) { + if (v->size == v->cap) { + size_t newcap = (v->cap > 0) ? v->cap * 2 : VEC_DEFAULT_INITCAP; + vecReserve(v, newcap); + } + + v->data[v->size++] = value; +} + +#ifdef REDIS_TEST + +#include +#include + +#include "testhelp.h" + +#define UNUSED(x) (void)(x) + +int vectorTest(int argc, char **argv, int flags) +{ + UNUSED(argc); + UNUSED(argv); + UNUSED(flags); + + vec v; + void *vstack[2]; + int one = 1, two = 2, three = 3, four = 4, five = 5, six = 6; + + vecInit(&v, vstack, 2); + test_cond("vecInit() stack-backed size is 0", vecSize(&v) == 0); + test_cond("vecInit() uses stack buffer", vecData(&v) == vstack); + vecReserve(&v, 1); + test_cond("vecReserve() no-ops when capacity is already sufficient", + v.cap == 2 && vecData(&v) == vstack); + vecPush(&v, &one); + vecPush(&v, &two); + test_cond("vecPush() appends into stack storage", + vecSize(&v) == 2 && vecData(&v) == vstack && + vecGet(&v, 0) == &one && vecGet(&v, 1) == &two); + vecReserve(&v, 4); + test_cond("vecReserve() spills from stack to heap preserving values", + v.cap == 4 && vecData(&v) != vstack && + vecGet(&v, 0) == &one && vecGet(&v, 1) == &two); + vecPush(&v, &three); + test_cond("vecPush() spills from stack to heap preserving values", + vecSize(&v) == 3 && + vecData(&v) != vstack && vecGet(&v, 0) == &one && + vecGet(&v, 1) == &two && vecGet(&v, 2) == &three); + + void **heap_data = vecData(&v); + vecClear(&v); + test_cond("vecClear() resets size but preserves storage", + vecSize(&v) == 0 && vecData(&v) == heap_data); + vecRelease(&v); + test_cond("vecRelease() resets vector state", + vecSize(&v) == 0 && vecData(&v) == NULL && v.cap == 0); + + vecInit(&v, NULL, 4); + test_cond("vecInit() heap-backed hint allocates storage", + vecSize(&v) == 0 && vecData(&v) != NULL && v.cap == 4); + vecPush(&v, &four); + test_cond("vecPush() works in heap-backed mode", + vecGet(&v, 0) == &four); + vecReserve(&v, 8); + test_cond("vecReserve() grows heap-backed storage preserving values", + v.cap == 8 && vecGet(&v, 0) == &four); + vecRelease(&v); + + vecInit(&v, NULL, 0); + vecReserve(&v, 6); + test_cond("vecReserve() allocates heap storage from empty vector", + v.cap == 6 && vecData(&v) != NULL); + vecPush(&v, &five); + vecPush(&v, &six); + test_cond("vecPush() works after vecReserve() on empty vector", + vecSize(&v) == 2 && + vecGet(&v, 0) == &five && vecGet(&v, 1) == &six); + vecRelease(&v); + + return 0; +} +#endif diff --git a/src/vector.h b/src/vector.h new file mode 100644 index 000000000..a3ea28505 --- /dev/null +++ b/src/vector.h @@ -0,0 +1,92 @@ +#ifndef REDIS_VECTOR_H +#define REDIS_VECTOR_H + +#include + +/* + * Simple append-only vector (dynamic array) of void * elements. + * + * Design: + * -------- + * - Stores elements in a contiguous array (void **). + * - Supports append (vecPush) and read access. + * - Optionally uses caller-provided stack buffer to avoid heap allocations. + * - See also comment in vector.c of vecInit() for more details. + * + * Memory: + * ------- + * - vecRelease() frees heap memory if used. + * - Stack buffer is never freed. + * - Stored elements are never freed. + * + * Modes: + * ------- + * 1. Start On Stack (grow to heap): vec v; + * void *vstack[8]; + * ... + * vecInit(&v, vstack, 8); + * + * Start Embedded (grow to heap): typedef struct { + * vec v; + * void *vembedded[8]; + * } obj; + * ... + * vecInit(&obj->v, obj->vembedded, 8); + * + * 2. Heap only, init capacity 8: vec v; + * ... + * vecInit(&v, NULL, 8); + * + * Heap only, init capacity 0: vec v; + * ... + * vecInit(&v, NULL, 0); + * + * 3. Depends on var size: vec v; + * void *vstack[8]; + * vecInit(&v, vstack, 8); + * vecReserve(&v, varsize); // varsize <= 8 ? stack : heap + * + * Notes: + * ------ + * - Not thread-safe. + * - If stack == NULL and initcap > 0, initcap is treated as an initial + * heap-capacity hint. + * - When used in Redis core, the implementation should use the Redis allocator + * wrappers (zmalloc / zrealloc / zfree) rather than libc allocation APIs. + */ + +typedef struct vec { + size_t size; /* Number of elements in the vector. */ + size_t cap; /* Capacity of the vector. */ + void **data; /* Heap-allocated storage or refers to stack. */ + void **stack; /* Optional stack buffer. */ +} vec; + +/* Initialize a vector */ +void vecInit(vec *v, void **stack, size_t initcap); + +/* Free only heap storage if any */ +void vecRelease(vec *v); + +/* Reset the logical length to zero while preserving allocated storage. */ +void vecClear(vec *v); + +size_t vecSize(const vec *v); + +/* Requires index < vecSize(v). */ +void *vecGet(const vec *v, size_t index); + +/* Return the contiguous backing array. */ +void **vecData(vec *v); + +/* Ensure capacity is at least mincap. */ +void vecReserve(vec *v, size_t mincap); + +/* Append one element, growing storage as needed. */ +void vecPush(vec *v, void *value); + +#ifdef REDIS_TEST +int vectorTest(int argc, char **argv, int flags); +#endif + +#endif /* REDIS_VECTOR_H */ diff --git a/tests/cluster/tests/18-info.tcl b/tests/cluster/tests/18-info.tcl index 68c62d357..744934990 100644 --- a/tests/cluster/tests/18-info.tcl +++ b/tests/cluster/tests/18-info.tcl @@ -41,5 +41,5 @@ test "errorstats: rejected call due to MOVED Redirection" { } assert_match {} [errorstat $pok MOVED] assert_match {*count=1*} [errorstat $perr MOVED] - assert_match {*calls=0,*,rejected_calls=1,failed_calls=0} [cmdstat $perr set] + assert_match {*calls=0,*,rejected_calls=1,failed_calls=0*} [cmdstat $perr set] } diff --git a/tests/helpers/gen_write_load.tcl b/tests/helpers/gen_write_load.tcl index 60d954e5d..e9f430ae1 100644 --- a/tests/helpers/gen_write_load.tcl +++ b/tests/helpers/gen_write_load.tcl @@ -18,7 +18,9 @@ set ::tlsdir "tests/tls" # Continuously sends SET commands to the server. If key is omitted, a random key # is used for every SET command. The value is always random. -proc gen_write_load {host port seconds tls {key ""} {size 0} {sleep 0}} { +# ignore_error_reply (default 0): when non-zero, MOVED/ASK replies are tolerated +# while draining pipelined responses (periodic 500-reply batches and final drain). +proc gen_write_load {host port seconds tls {key ""} {size 0} {sleep 0} {ignore_error_reply 0}} { set start_time [clock seconds] set r [redis $host $port 1 $tls] $r client setname LOAD_HANDLER @@ -44,12 +46,19 @@ proc gen_write_load {host port seconds tls {key ""} {size 0} {sleep 0}} { } else { $r set $key $value } - + incr count if {$count % 500 == 0} { for {set i 0} {$i < 500} {incr i} { - $r read + # Capture opts to preserve original errorInfo/errorCode on re-raise. + if {[catch {$r read} err opts]} { + if {$ignore_error_reply && ([string match {MOVED*} $err] || [string match {ASK*} $err])} { + continue + } + return -options $opts $err + } } + set count 0 } if {[clock seconds]-$start_time > $seconds} { @@ -59,12 +68,17 @@ proc gen_write_load {host port seconds tls {key ""} {size 0} {sleep 0}} { after $sleep } } - + # Read remaining replies for {set i 0} {$i < $count} {incr i} { - $r read + if {[catch {$r read} err opts]} { + if {$ignore_error_reply && ([string match {MOVED*} $err] || [string match {ASK*} $err])} { + continue + } + return -options $opts $err + } } exit 0 } -gen_write_load [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] [lindex $argv 4] [lindex $argv 5] [lindex $argv 6] +gen_write_load [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] [lindex $argv 4] [lindex $argv 5] [lindex $argv 6] [lindex $argv 7] diff --git a/tests/integration/dismiss-mem.tcl b/tests/integration/dismiss-mem.tcl index 6e790665a..2b0fbb3e4 100644 --- a/tests/integration/dismiss-mem.tcl +++ b/tests/integration/dismiss-mem.tcl @@ -4,7 +4,7 @@ # Actually, we may not have many asserts in the test, since we just check for # crashes and the dump file inconsistencies. -start_server {tags {"dismiss external:skip"}} { +start_server {tags {"dismiss external:skip needs:debug"}} { # In other tests, although we test child process dumping RDB file, but # memory allocations of key/values are usually small, they couldn't cover # the "dismiss" object methods, in this test, we create big size key/values @@ -47,12 +47,15 @@ start_server {tags {"dismiss external:skip"}} { r xadd bigstream * entry1 $bigstr entry2 $bigstr set digest [debug_digest] - r config set aof-use-rdb-preamble no - r bgrewriteaof - waitForBgrewriteaof r - r debug loadaof - set newdigest [debug_digest] - assert {$digest eq $newdigest} + # Test both RDB (yes) and AOF (no) rewrite paths. + foreach preamble {yes no} { + r config set aof-use-rdb-preamble $preamble + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + set newdigest [debug_digest] + assert {$digest eq $newdigest} + } } test {dismiss client output buffer} { @@ -99,4 +102,48 @@ start_server {tags {"dismiss external:skip"}} { waitForBgsave $master } } + + test {dismiss multi-db kvstore bucket memory in standalone mode} { + r flushall + regexp {db=(\d+)} [r client info] -> curdb + # Populate multiple DBs to verify each DB's bucket arrays can be dismissed. + foreach db {0 1 2 3} { + r select $db + populate 2000 "db${db}key:" 3 0 false 3600 + } + set digest [debug_digest] + + # Test both RDB (yes) and AOF (no) rewrite paths. + foreach preamble {yes no} { + r config set aof-use-rdb-preamble $preamble + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + set newdigest [debug_digest] + assert {$digest eq $newdigest} + } + r select $curdb + } +} + +start_cluster 1 0 {tags {dismiss external:skip cluster needs:debug}} { + test {dismiss slot dict bucket memory in cluster mode} { + # Concentrate keys into a few slots using hash tags so each slot's + # bucket array is large enough to be dismissed. + # {06S} -> slot 0, {Qi} -> slot 1, {5L5} -> slot 2 + foreach tag {{06S} {Qi} {5L5}} { + populate 2000 "${tag}key:" 3 0 false 3600 + } + set digest [r debug digest] + + # Test both RDB (yes) and AOF (no) rewrite paths. + foreach preamble {yes no} { + r config set aof-use-rdb-preamble $preamble + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + set newdigest [r debug digest] + assert {$digest eq $newdigest} + } + } } diff --git a/tests/support/util.tcl b/tests/support/util.tcl index 0c9f64836..16eb80008 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -604,9 +604,11 @@ proc find_valgrind_errors {stderr on_termination} { # Execute a background process writing random data for the specified number # of seconds to the specified Redis instance. If key is omitted, a random key # is used for every SET command. -proc start_write_load {host port seconds {key ""} {size 0} {sleep 0}} { +# ignore_error_reply (default 0): set non-zero in cluster slot-migration tests to tolerate +# MOVED/ASK replies while draining pipelined writes in the load helper. +proc start_write_load {host port seconds {key ""} {size 0} {sleep 0} {ignore_error_reply 0}} { set tclsh [info nameofexecutable] - exec $tclsh tests/helpers/gen_write_load.tcl $host $port $seconds $::tls $key $size $sleep & + exec $tclsh tests/helpers/gen_write_load.tcl $host $port $seconds $::tls $key $size $sleep $ignore_error_reply & } # Stop a process generating write load executed with start_write_load. diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index 6f9d94f29..77bb37095 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -357,7 +357,7 @@ start_server {tags {"acl external:skip"}} { assert_error {*NOPERM No permissions to access a key*} {$rd read} $rd ping $rd close - assert_match {*calls=0,usec=0,*,rejected_calls=1,failed_calls=0} [cmdrstat blpop r] + assert_match {*calls=0,usec=0,*,rejected_calls=1,failed_calls=0*} [cmdrstat blpop r] } test {Users can be configured to authenticate with any password} { diff --git a/tests/unit/cluster/atomic-slot-migration.tcl b/tests/unit/cluster/atomic-slot-migration.tcl index 826f0d69c..74eee55f0 100644 --- a/tests/unit/cluster/atomic-slot-migration.tcl +++ b/tests/unit/cluster/atomic-slot-migration.tcl @@ -577,23 +577,16 @@ start_cluster 3 3 {tags {external:skip cluster} overrides {cluster-node-timeout R 1 debug asm-trim-method none populate_slot 10000 -idx 1 -slot 6000 - # Start write traffic on node-0 - # Throws -MOVED error once asm is completed, catch block will ignore it. - catch { - # Start the slot 0 write load on the R 0 - set port [get_port 0] - set key [slot_key 0 mykey] - set load_handle0 [start_write_load "127.0.0.1" $port 100 $key 0 5] - } + # Start write traffic on node-0 (ignore_error_reply=1 tolerates MOVED/ASK + # replies while slots are being migrated). + set port [get_port 0] + set key [slot_key 0 mykey] + set load_handle0 [start_write_load "127.0.0.1" $port 100 $key 0 5 1] - # Start write traffic on node-1 - # Throws -MOVED error once asm is completed, catch block will ignore it. - catch { - # Start the slot 6000 write load on the R 1 - set port [get_port 1] - set key [slot_key 6000 mykey] - set load_handle1 [start_write_load "127.0.0.1" $port 100 $key 0 5] - } + # Start write traffic on node-1 (ignore_error_reply=1 for migration redirects). + set port [get_port 1] + set key [slot_key 6000 mykey] + set load_handle1 [start_write_load "127.0.0.1" $port 100 $key 0 5 1] # Migrate keys R 1 CLUSTER MIGRATION IMPORT 0 100 @@ -801,8 +794,9 @@ start_cluster 3 3 {tags {external:skip cluster} overrides {cluster-node-timeout # we set a delay to write incremental data R 1 config set rdb-key-save-delay 1000000 - # Start the slot 0 write load on the R 1 - set load_handle [start_write_load "127.0.0.1" [get_port 1] 100 $slot0_key] + # Start slot 0 write load on R1. ignore_error_reply=1 tolerates MOVED/ASK + # replies that can appear while slot 0 is being migrated. + set load_handle [start_write_load "127.0.0.1" [get_port 1] 100 $slot0_key 0 0 1] # Clear all fail points assert_equal {OK} [R 0 debug asm-failpoint "" ""] diff --git a/tests/unit/gcra.tcl b/tests/unit/gcra.tcl index e11101484..b012a0fc4 100644 --- a/tests/unit/gcra.tcl +++ b/tests/unit/gcra.tcl @@ -229,6 +229,103 @@ start_server {tags {"gcra" "external:skip"}} { } } +start_server {tags {"gcra" "external:skip"}} { + test {GCRA - RDB save and reload preserves value} { + r del mykey + r gcra mykey 5 1 60 + r gcra mykey 5 1 60 + + set dump_before [r dump mykey] + + r debug reload + + assert_equal [r type mykey] "gcra" + set dump_after [r dump mykey] + assert_equal $dump_before $dump_after + } {} {needs:debug} + + test {GCRA - RDB save and reload preserves TTL} { + r del mykey + r gcra mykey 5 1 60 + set ttl_before [r pexpiretime mykey] + assert_morethan $ttl_before 0 + + r debug reload + + set ttl_after [r pexpiretime mykey] + assert_morethan $ttl_after 0 + assert_equal $ttl_after $ttl_before + } {} {needs:debug} + + test {GCRA - DUMP and RESTORE roundtrip} { + r del mykey mykey2 + r gcra mykey 5 1 60 + r gcra mykey 5 1 60 + + set dump [r dump mykey] + set ttl [r pttl mykey] + r restore mykey2 $ttl $dump + + assert_equal [r type mykey2] "gcra" + + set result_orig [r gcra mykey 5 1 60] + set result_restored [r gcra mykey2 5 1 60] + assert_equal [lindex $result_orig 2] [lindex $result_restored 2] + } + + test {GCRA - AOF rewrite preserves value} { + r del mykey + r config set appendonly yes + waitForBgrewriteaof r + + r gcra mykey 5 1 60 + r gcra mykey 5 1 60 + + set dump_before [r dump mykey] + + r BGREWRITEAOF + waitForBgrewriteaof r + r debug reload + + assert_equal [r type mykey] "gcra" + set dump_after [r dump mykey] + assert_equal $dump_before $dump_after + } {} {external:skip needs:debug} + + test {GCRA - AOF rewrite preserves TTL} { + r del mykey + r config set appendonly yes + waitForBgrewriteaof r + + r gcra mykey 5 1 60 + + r BGREWRITEAOF + waitForBgrewriteaof r + + set ttl_before [r pttl mykey] + assert {$ttl_before > 0} + + r debug reload + + set ttl_after [r pttl mykey] + assert {$ttl_after > 0} + assert {$ttl_after <= $ttl_before} + } {} {external:skip needs:debug} + + test {GCRA - DEBUG DIGEST consistent after RDB reload} { + r del mykey + r gcra mykey 5 1 60 + r gcra mykey 5 1 60 + + set digest_before [r debug digest] + + r debug reload + + set digest_after [r debug digest] + assert_equal $digest_before $digest_after + } {} {needs:debug} +} + start_server {tags {"gcra repl" "external:skip"}} { set replica [srv 0 client] set replica_host [srv 0 host] @@ -240,27 +337,26 @@ start_server {tags {"gcra repl" "external:skip"}} { set master_host [srv 0 host] set master_port [srv 0 port] - $master flushdb - $replica flushdb + test {GCRA - Replication works} { + $master flushdb + $replica flushdb - $replica replicaof $master_host $master_port - wait_for_condition 100 100 { - [s -1 master_link_status] eq "up" - } else { - fail "Master <-> Replica didn't finish sync" - } + $replica replicaof $master_host $master_port + wait_for_condition 100 100 { + [s -1 master_link_status] eq "up" + } else { + fail "Master <-> Replica didn't finish sync" + } - set cmdinfo [$replica info commandstats] - assert_equal [lsearch -glob $cmdinfo "cmdstat_gcra:*"] -1 - assert_equal [lsearch -glob $cmdinfo "cmdstat_set:*"] -1 + set cmdinfo [$replica info commandstats] + assert_equal [lsearch -glob $cmdinfo "cmdstat_gcrasetvalue:*"] -1 - $master del mykey - $master gcra mykey 2 1 1000 TOKENS 2 + $master del mykey + $master gcra mykey 2 1 1000 TOKENS 2 + wait_for_ofs_sync $master $replica - wait_for_ofs_sync $master $replica - - set cmdinfo [$replica info commandstats] - assert_equal [lsearch -glob $cmdinfo "cmdstat_gcra:*"] -1 - assert_morethan_equal [lsearch -glob $cmdinfo "cmdstat_set:*"] 0 + set cmdinfo [$replica info commandstats] + assert_morethan_equal [lsearch -glob $cmdinfo "cmdstat_gcrasetvalue:*"] 0 + } {} {external:skip} } } diff --git a/tests/unit/info.tcl b/tests/unit/info.tcl index 0dee39d55..07543b3ad 100644 --- a/tests/unit/info.tcl +++ b/tests/unit/info.tcl @@ -121,7 +121,7 @@ start_server {tags {"info" "external:skip"}} { catch {r auth k} e assert_match {ERR AUTH*} $e assert_match {*count=1*} [errorstat ERR] - assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1*} [cmdstat auth] assert_equal [s total_error_replies] 1 r config resetstat assert_match {} [errorstat ERR] @@ -137,15 +137,15 @@ start_server {tags {"info" "external:skip"}} { catch {r exec} e assert_match {ERR AUTH*} $e assert_match {*count=1*} [errorstat ERR] - assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdstat set] - assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat auth] - assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdstat exec] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=0*} [cmdstat set] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1*} [cmdstat auth] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=0*} [cmdstat exec] assert_equal [s total_error_replies] 1 # MULTI/EXEC command errors should still be pinpointed to him catch {r exec} e assert_match {ERR EXEC without MULTI} $e - assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat exec] + assert_match {*calls=2,*,rejected_calls=0,failed_calls=1*} [cmdstat exec] assert_match {*count=2*} [errorstat ERR] assert_equal [s total_error_replies] 2 } @@ -174,7 +174,7 @@ start_server {tags {"info" "external:skip"}} { catch {r evalsha NotValidShaSUM 0} e assert_match {NOSCRIPT*} $e assert_match {*count=1*} [errorstat NOSCRIPT] - assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat evalsha] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1*} [cmdstat evalsha] assert_equal [s total_error_replies] 1 r config resetstat assert_match {} [errorstat NOSCRIPT] @@ -188,7 +188,7 @@ start_server {tags {"info" "external:skip"}} { catch {r XGROUP CREATECONSUMER mystream mygroup consumer} e assert_match {NOGROUP*} $e assert_match {*count=1*} [errorstat NOGROUP] - assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat xgroup\\|createconsumer] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1*} [cmdstat xgroup\\|createconsumer] r config resetstat assert_match {} [errorstat NOGROUP] } @@ -217,9 +217,9 @@ start_server {tags {"info" "external:skip"}} { assert_match {*count=1*} [errorstat ERR] assert_match {*count=1*} [errorstat EXECABORT] assert_equal [s total_error_replies] 2 - assert_match {*calls=0,*,rejected_calls=1,failed_calls=0} [cmdstat set] - assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdstat multi] - assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat exec] + assert_match {*calls=0,*,rejected_calls=1,failed_calls=0*} [cmdstat set] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=0*} [cmdstat multi] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1*} [cmdstat exec] assert_equal [s total_error_replies] 2 r config resetstat assert_match {} [errorstat ERR] @@ -232,11 +232,11 @@ start_server {tags {"info" "external:skip"}} { catch {r set k} e assert_match {ERR wrong number of arguments for 'set' command} $e assert_match {*count=1*} [errorstat ERR] - assert_match {*calls=0,*,rejected_calls=1,failed_calls=0} [cmdstat set] + assert_match {*calls=0,*,rejected_calls=1,failed_calls=0*} [cmdstat set] # ensure that after a rejected command, valid ones are counted properly r set k1 v1 r set k2 v2 - assert_match {calls=2,*,rejected_calls=1,failed_calls=0} [cmdstat set] + assert_match {calls=2,*,rejected_calls=1,failed_calls=0*} [cmdstat set] assert_equal [s total_error_replies] 1 } @@ -248,7 +248,7 @@ start_server {tags {"info" "external:skip"}} { catch {r set a b} e assert_match {OOM*} $e assert_match {*count=1*} [errorstat OOM] - assert_match {*calls=0,*,rejected_calls=1,failed_calls=0} [cmdstat set] + assert_match {*calls=0,*,rejected_calls=1,failed_calls=0*} [cmdstat set] assert_equal [s total_error_replies] 1 r config resetstat assert_match {} [errorstat OOM] @@ -264,7 +264,7 @@ start_server {tags {"info" "external:skip"}} { catch {r set a b} e assert_match {NOPERM*} $e assert_match {*count=1*} [errorstat NOPERM] - assert_match {*calls=0,*,rejected_calls=1,failed_calls=0} [cmdstat set] + assert_match {*calls=0,*,rejected_calls=1,failed_calls=0*} [cmdstat set] assert_equal [s total_error_replies] 1 r config resetstat assert_match {} [errorstat NOPERM] @@ -283,7 +283,7 @@ start_server {tags {"info" "external:skip"}} { r client unblock $rd_id error assert_error {UNBLOCKED*} {$rd read} assert_match {*count=1*} [errorstat UNBLOCKED] - assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat blpop] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1*} [cmdstat blpop] assert_equal [s total_error_replies] 1 $rd close } diff --git a/tests/unit/moduleapi/blockedclient.tcl b/tests/unit/moduleapi/blockedclient.tcl index 7dcc1d6a6..71aafd444 100644 --- a/tests/unit/moduleapi/blockedclient.tcl +++ b/tests/unit/moduleapi/blockedclient.tcl @@ -248,16 +248,16 @@ foreach call_type {nested normal} { # RM_Call that propagates an error assert_error "WRONGTYPE*" {r do_rm_call hgetall x} assert_equal [errorrstat WRONGTYPE r] {count=1} - assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdrstat hgetall r] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1*} [cmdrstat hgetall r] # RM_Call from bg thread that propagates an error assert_error "WRONGTYPE*" {r do_bg_rm_call hgetall x} assert_equal [errorrstat WRONGTYPE r] {count=2} - assert_match {*calls=2,*,rejected_calls=0,failed_calls=2} [cmdrstat hgetall r] + assert_match {*calls=2,*,rejected_calls=0,failed_calls=2*} [cmdrstat hgetall r] assert_equal [s total_error_replies] 6 - assert_match {*calls=5,*,rejected_calls=0,failed_calls=4} [cmdrstat do_rm_call r] - assert_match {*calls=2,*,rejected_calls=0,failed_calls=2} [cmdrstat do_bg_rm_call r] + assert_match {*calls=5,*,rejected_calls=0,failed_calls=4*} [cmdrstat do_rm_call r] + assert_match {*calls=2,*,rejected_calls=0,failed_calls=2*} [cmdrstat do_bg_rm_call r] } set master [srv 0 client] diff --git a/tests/unit/moduleapi/moduleauth.tcl b/tests/unit/moduleapi/moduleauth.tcl index f8b721ce1..16951175c 100644 --- a/tests/unit/moduleapi/moduleauth.tcl +++ b/tests/unit/moduleapi/moduleauth.tcl @@ -36,15 +36,15 @@ start_server {tags {"modules external:skip"}} { r acl setuser foo >pwd on ~* &* +@all assert_equal {OK} [r AUTH foo allow] assert_error {*Auth denied by Misc Module*} {r AUTH foo deny} - assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + assert_match {*calls=2,*,rejected_calls=0,failed_calls=1*} [cmdstat auth] assert_error {*WRONGPASS*} {r AUTH foo nomatch} - assert_match {*calls=3,*,rejected_calls=0,failed_calls=2} [cmdstat auth] + assert_match {*calls=3,*,rejected_calls=0,failed_calls=2*} [cmdstat auth] assert_equal {OK} [r AUTH foo pwd] # Test for No Pass user r acl setuser foo on ~* &* +@all nopass assert_equal {OK} [r AUTH foo allow] assert_error {*Auth denied by Misc Module*} {r AUTH foo deny} - assert_match {*calls=6,*,rejected_calls=0,failed_calls=3} [cmdstat auth] + assert_match {*calls=6,*,rejected_calls=0,failed_calls=3*} [cmdstat auth] assert_equal {OK} [r AUTH foo nomatch] # Validate that the Module added an ACL Log entry. @@ -67,13 +67,13 @@ start_server {tags {"modules external:skip"}} { assert_equal $hello3_response [r HELLO 3 AUTH foo allow] # Validate denying AUTH for the HELLO cmd assert_error {*Auth denied by Misc Module*} {r HELLO 2 AUTH foo deny} - assert_match {*calls=5,*,rejected_calls=0,failed_calls=1} [cmdstat hello] + assert_match {*calls=5,*,rejected_calls=0,failed_calls=1*} [cmdstat hello] assert_error {*WRONGPASS*} {r HELLO 2 AUTH foo nomatch} - assert_match {*calls=6,*,rejected_calls=0,failed_calls=2} [cmdstat hello] + assert_match {*calls=6,*,rejected_calls=0,failed_calls=2*} [cmdstat hello] assert_error {*Auth denied by Misc Module*} {r HELLO 3 AUTH foo deny} - assert_match {*calls=7,*,rejected_calls=0,failed_calls=3} [cmdstat hello] + assert_match {*calls=7,*,rejected_calls=0,failed_calls=3*} [cmdstat hello] assert_error {*WRONGPASS*} {r HELLO 3 AUTH foo nomatch} - assert_match {*calls=8,*,rejected_calls=0,failed_calls=4} [cmdstat hello] + assert_match {*calls=8,*,rejected_calls=0,failed_calls=4*} [cmdstat hello] # Validate that the Module added an ACL Log entry. set entry [lindex [r ACL LOG] 1] @@ -97,10 +97,10 @@ start_server {tags {"modules external:skip"}} { r client setname client0 assert_error {*Auth denied by Misc Module*} {r HELLO 2 AUTH foo deny setname client1} assert {[r client getname] eq {client0}} - assert_match {*calls=3,*,rejected_calls=0,failed_calls=1} [cmdstat hello] + assert_match {*calls=3,*,rejected_calls=0,failed_calls=1*} [cmdstat hello] assert_error {*WRONGPASS*} {r HELLO 2 AUTH foo nomatch setname client2} assert {[r client getname] eq {client0}} - assert_match {*calls=4,*,rejected_calls=0,failed_calls=2} [cmdstat hello] + assert_match {*calls=4,*,rejected_calls=0,failed_calls=2*} [cmdstat hello] } test {test blocking module AUTH} { @@ -109,15 +109,15 @@ start_server {tags {"modules external:skip"}} { r acl setuser foo >pwd on ~* &* +@all assert_equal {OK} [r AUTH foo block_allow] assert_error {*Auth denied by Misc Module*} {r AUTH foo block_deny} - assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + assert_match {*calls=2,*,rejected_calls=0,failed_calls=1*} [cmdstat auth] assert_error {*WRONGPASS*} {r AUTH foo nomatch} - assert_match {*calls=3,*,rejected_calls=0,failed_calls=2} [cmdstat auth] + assert_match {*calls=3,*,rejected_calls=0,failed_calls=2*} [cmdstat auth] assert_equal {OK} [r AUTH foo pwd] # Test for No Pass user r acl setuser foo on ~* &* +@all nopass assert_equal {OK} [r AUTH foo block_allow] assert_error {*Auth denied by Misc Module*} {r AUTH foo block_deny} - assert_match {*calls=6,*,rejected_calls=0,failed_calls=3} [cmdstat auth] + assert_match {*calls=6,*,rejected_calls=0,failed_calls=3*} [cmdstat auth] assert_equal {OK} [r AUTH foo nomatch] # Validate that every Blocking AUTH command took at least 500000 usec. set stats [cmdstat auth] @@ -144,13 +144,13 @@ start_server {tags {"modules external:skip"}} { assert_equal $hello3_response [r HELLO 3 AUTH foo block_allow] # validate denying AUTH for the HELLO cmd assert_error {*Auth denied by Misc Module*} {r HELLO 2 AUTH foo block_deny} - assert_match {*calls=5,*,rejected_calls=0,failed_calls=1} [cmdstat hello] + assert_match {*calls=5,*,rejected_calls=0,failed_calls=1*} [cmdstat hello] assert_error {*WRONGPASS*} {r HELLO 2 AUTH foo nomatch} - assert_match {*calls=6,*,rejected_calls=0,failed_calls=2} [cmdstat hello] + assert_match {*calls=6,*,rejected_calls=0,failed_calls=2*} [cmdstat hello] assert_error {*Auth denied by Misc Module*} {r HELLO 3 AUTH foo block_deny} - assert_match {*calls=7,*,rejected_calls=0,failed_calls=3} [cmdstat hello] + assert_match {*calls=7,*,rejected_calls=0,failed_calls=3*} [cmdstat hello] assert_error {*WRONGPASS*} {r HELLO 3 AUTH foo nomatch} - assert_match {*calls=8,*,rejected_calls=0,failed_calls=4} [cmdstat hello] + assert_match {*calls=8,*,rejected_calls=0,failed_calls=4*} [cmdstat hello] # Validate that every HELLO AUTH command took at least 500000 usec. set stats [cmdstat hello] regexp "usec_per_call=(\[0-9]{1,})\.*," $stats all usec_per_call @@ -178,10 +178,10 @@ start_server {tags {"modules external:skip"}} { r client setname client0 assert_error {*Auth denied by Misc Module*} {r HELLO 2 AUTH foo block_deny setname client1} assert {[r client getname] eq {client0}} - assert_match {*calls=3,*,rejected_calls=0,failed_calls=1} [cmdstat hello] + assert_match {*calls=3,*,rejected_calls=0,failed_calls=1*} [cmdstat hello] assert_error {*WRONGPASS*} {r HELLO 2 AUTH foo nomatch setname client2} assert {[r client getname] eq {client0}} - assert_match {*calls=4,*,rejected_calls=0,failed_calls=2} [cmdstat hello] + assert_match {*calls=4,*,rejected_calls=0,failed_calls=2*} [cmdstat hello] # Validate that every HELLO AUTH SETNAME command took at least 500000 usec. set stats [cmdstat hello] regexp "usec_per_call=(\[0-9]{1,})\.*," $stats all usec_per_call @@ -205,7 +205,7 @@ start_server {tags {"modules external:skip"}} { # Case 2 - Non Blocking Deny assert_error {*Auth denied by Misc Module*} {r AUTH foo deny} - assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + assert_match {*calls=2,*,rejected_calls=0,failed_calls=1*} [cmdstat auth] r config resetstat @@ -214,7 +214,7 @@ start_server {tags {"modules external:skip"}} { # Case 4 - Blocking Deny assert_error {*Auth denied by Misc Module*} {r AUTH foo block_deny} - assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + assert_match {*calls=2,*,rejected_calls=0,failed_calls=1*} [cmdstat auth] # Validate that every Blocking AUTH command took at least 500000 usec. set stats [cmdstat auth] @@ -228,13 +228,13 @@ start_server {tags {"modules external:skip"}} { # Case 6 - Non Blocking Deny via the second module. assert_error {*Auth denied by Misc Module*} {r AUTH foo deny_two} - assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + assert_match {*calls=2,*,rejected_calls=0,failed_calls=1*} [cmdstat auth] r config resetstat # Case 7 - All four auth callbacks "Skip" by not explicitly allowing or denying. assert_error {*WRONGPASS*} {r AUTH foo nomatch} - assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1*} [cmdstat auth] assert_equal {OK} [r AUTH foo pwd] # Because we had to attempt all 4 callbacks, validate that the AUTH command took at least @@ -283,7 +283,7 @@ start_server {tags {"modules external:skip"}} { r multi r AUTH foo block_allow assert_error {*ERR Blocking module command called from transaction*} {r exec} - assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + assert_match {*calls=2,*,rejected_calls=0,failed_calls=1*} [cmdstat auth] } test {Disabling Redis User during blocking module auth} { @@ -300,7 +300,7 @@ start_server {tags {"modules external:skip"}} { wait_for_blocked_clients_count 0 500 10 $rd flush assert_error {*WRONGPASS*} { $rd read } - assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1*} [cmdstat auth] } test {Killing a client in the middle of blocking module auth} { @@ -354,7 +354,7 @@ start_server {tags {"modules external:skip"}} { $rd flush assert_equal [$rd read] "OK" set stats [cmdstat auth] - assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} $stats + assert_match {*calls=1,*,rejected_calls=0,failed_calls=0*} $stats # Validate that even the new blocking module auth cb which was registered in the middle of # blocking module auth is attempted - making it take twice the duration (2x 500000 us). @@ -387,7 +387,7 @@ start_server {tags {"modules external:skip"}} { wait_for_blocked_clients_count 0 500 10 $rd flush assert_equal [$rd read] "OK" - assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdstat auth] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=0*} [cmdstat auth] # Validate that unloading the moduleauthtwo module does not unregister module auth cbs of # of the testacl module. Module based auth should succeed. @@ -400,6 +400,6 @@ start_server {tags {"modules external:skip"}} { assert_error {*WRONGPASS*} {r AUTH foo block_allow} assert_error {*WRONGPASS*} {r AUTH foo allow_two} assert_error {*WRONGPASS*} {r AUTH foo allow} - assert_match {*calls=5,*,rejected_calls=0,failed_calls=3} [cmdstat auth] + assert_match {*calls=5,*,rejected_calls=0,failed_calls=3*} [cmdstat auth] } } diff --git a/tests/unit/sort.tcl b/tests/unit/sort.tcl index 35ec1606e..0dee25b29 100644 --- a/tests/unit/sort.tcl +++ b/tests/unit/sort.tcl @@ -204,6 +204,14 @@ foreach command {SORT SORT_RO} { assert_equal [lsort -real $floats] [r sort mylist] } + test "SORT BY with smallest normal double 2.2250738585072012e-308" { + r flushdb + r lpush mylist a b + r set weight_a 2.2250738585072012e-308 + r set weight_b 1 + assert_equal {a b} [r sort mylist BY weight_*] + } {} {cluster:skip} + test "SORT with STORE returns zero if result is empty (github issue 224)" { r flushdb r sort foo{t} store bar{t} diff --git a/tests/unit/type/hash-field-expire.tcl b/tests/unit/type/hash-field-expire.tcl index 7f3520e80..402a9ad72 100644 --- a/tests/unit/type/hash-field-expire.tcl +++ b/tests/unit/type/hash-field-expire.tcl @@ -1277,6 +1277,24 @@ start_server {tags {"external:skip needs:debug"}} { assert_range [r hpttl myhash FIELDS 1 f3] 4500 5000 } + test "HSETEX EX - field appears twice in FIELDS list with EX is allowed ($type)" { + # The EX condition passes, so all fields must be set, and the last value wins. + r del myhash + r hset myhash f1 v1 + r hsetex myhash EX 100 FIELDS 2 f1 new1 f1 new2 + # Last value wins (same as plain HSET behavior with duplicate fields) + assert_equal "new2" [r hget myhash f1] + assert_range [r httl myhash FIELDS 1 f1] 80 100 + } + + test "HSETEX FNX - field appears twice in FIELDS list with EX is allowed ($type)" { + # The FNX condition passes, so all fields must be set, and the last value wins. + r del myhash + r hsetex myhash FNX EX 100 FIELDS 2 f1 new1 f1 new2 + assert_equal "new2" [r hget myhash f1] + assert_range [r httl myhash FIELDS 1 f1] 80 100 + } + test "HSETEX - Test 'EX' flag ($type)" { r del myhash r hset myhash f1 v1 f2 v2 @@ -2359,6 +2377,11 @@ start_server {tags {"hash"}} { assert_error {*Parameter*numFields*should be greater than 0*} {r HEXPIRE myhash 60 FIELDS -1 f1} assert_error {*invalid number of fields*} {r HSETEX myhash FIELDS 0 f1 v1 EX 60} assert_error {*invalid number of fields*} {r HGETEX myhash FIELDS 0 f1 EX 60} + set future_sec [expr {[clock seconds] + 60}] + set future_ms [expr {[clock milliseconds] + 60000}] + foreach {cmd expire} [list HEXPIRE 60 HPEXPIRE 60000 HEXPIREAT $future_sec HPEXPIREAT $future_ms] { + assert_error {*wrong number of arguments*} [list r $cmd myhash $expire FIELDS 2147483647 f1] + } # Test missing FIELDS keyword assert_error {*unknown argument*} {r HEXPIRE myhash 60 2 f1 f2} diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl index 8d17c5c38..96f80e9fb 100644 --- a/tests/unit/type/list.tcl +++ b/tests/unit/type/list.tcl @@ -2366,7 +2366,7 @@ foreach {pop} {BLPOP BLMPOP_RIGHT} { r LPUSH mylist 1 wait_for_blocked_clients_count 0 - assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdrstat blpop r] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=0*} [cmdrstat blpop r] $rd close } @@ -2390,7 +2390,7 @@ foreach {pop} {BLPOP BLMPOP_RIGHT} { # unblock the client on timeout r client unblock $id timeout - assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdrstat blpop r] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=0*} [cmdrstat blpop r] $rd close } diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index 9adb7c705..60e40596b 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -690,7 +690,7 @@ start_server { # verify command stats, error stats and error counter work on failed blocked command assert_match {*count=1*} [errorrstat NOGROUP r] - assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdrstat xreadgroup r] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1*} [cmdrstat xreadgroup r] assert_equal [s total_error_replies] 1 } @@ -1905,6 +1905,79 @@ start_server { } } + start_server {tags {"repl external:skip" "stream"}} { + # Verify that XREADGROUP propagates a newly created consumer to + # the replica in cases where no XCLAIM is generated (XCLAIM + # implicitly creates the consumer, so explicit propagation is + # only needed when it is absent). Two cases are tested: + # 1. Without NOACK and no messages to deliver — no XCLAIM at all. + # 2. With NOACK and messages delivered — NOACK skips PEL/XCLAIM. + test "XREADGROUP propagates new consumer to replica" { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + + start_server {tags {"stream"}} { + set replica [srv 0 client] + + $replica replicaof $master_host $master_port + wait_for_sync $replica + + $master DEL mystream + $master XADD mystream 1-0 f v + $master XGROUP CREATE mystream grp 0 + + # Consume the only message so the stream has no + # new messages pending for delivery. + $master XREADGROUP GROUP grp c1 STREAMS mystream > + $master XACK mystream grp 1-0 + + wait_for_ofs_sync $master $replica + + # Case 1: XREADGROUP without NOACK for a brand-new + # consumer when there are NO messages to deliver. + # No XCLAIM is generated, so the consumer must be + # explicitly propagated. + set reply [$master XREADGROUP GROUP grp c2 STREAMS mystream >] + assert_equal $reply {} + + set master_consumers [$master XINFO CONSUMERS mystream grp] + set master_names [lmap c $master_consumers {dict get $c name}] + assert {[lsearch $master_names "c2"] >= 0} + + wait_for_ofs_sync $master $replica + + set replica_consumers [$replica XINFO CONSUMERS mystream grp] + set replica_names [lmap c $replica_consumers {dict get $c name}] + if {[lsearch $replica_names "c2"] < 0} { + fail "Consumer 'c2' not found on replica (have: $replica_names)" + } + + # Case 2: XREADGROUP with NOACK for a brand-new consumer + # when a message IS available. NOACK skips PEL/XCLAIM + # entirely, so the consumer must be explicitly propagated + # even though messages were delivered. + $master XADD mystream 2-0 f v + wait_for_ofs_sync $master $replica + + set reply [$master XREADGROUP GROUP grp c3 NOACK STREAMS mystream >] + assert {$reply ne {}} + + set master_consumers [$master XINFO CONSUMERS mystream grp] + set master_names [lmap c $master_consumers {dict get $c name}] + assert {[lsearch $master_names "c3"] >= 0} + + wait_for_ofs_sync $master $replica + + set replica_consumers [$replica XINFO CONSUMERS mystream grp] + set replica_names [lmap c $replica_consumers {dict get $c name}] + if {[lsearch $replica_names "c3"] < 0} { + fail "Consumer 'c3' not found on replica (have: $replica_names)" + } + } + } + } + start_server {} { if {!$::force_resp3} { test "XREADGROUP CLAIM field types are correct" { diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index ad9483b2d..f08ddf70c 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -971,6 +971,26 @@ start_server {tags {"zset"}} { assert_equal {b 2 c 3} [r zinter 2 zseta{t} zsetb{t} aggregate max withscores] } + test "ZUNIONSTORE with AGGREGATE COUNT - $encoding" { + assert_equal 4 [r zunionstore zsetc{t} 2 zseta{t} zsetb{t} aggregate count] + assert_equal {a 1 d 1 b 2 c 2} [r zrange zsetc{t} 0 -1 withscores] + } + + test "ZUNION/ZINTER with AGGREGATE COUNT - $encoding" { + assert_equal {a 1 d 1 b 2 c 2} [r zunion 2 zseta{t} zsetb{t} aggregate count withscores] + assert_equal {b 2 c 2} [r zinter 2 zseta{t} zsetb{t} aggregate count withscores] + } + + test "ZUNIONSTORE with AGGREGATE COUNT and WEIGHTS - $encoding" { + assert_equal 4 [r zunionstore zsetc{t} 2 zseta{t} zsetb{t} weights 2 3 aggregate count] + assert_equal {a 2 d 3 b 5 c 5} [r zrange zsetc{t} 0 -1 withscores] + } + + test "ZUNION/ZINTER with AGGREGATE COUNT and WEIGHTS - $encoding" { + assert_equal {a 2 d 3 b 5 c 5} [r zunion 2 zseta{t} zsetb{t} weights 2 3 aggregate count withscores] + assert_equal {b 5 c 5} [r zinter 2 zseta{t} zsetb{t} weights 2 3 aggregate count withscores] + } + test "ZINTERSTORE basics - $encoding" { assert_equal 2 [r zinterstore zsetc{t} 2 zseta{t} zsetb{t}] assert_equal {b 3 c 5} [r zrange zsetc{t} 0 -1 withscores] @@ -1030,6 +1050,39 @@ start_server {tags {"zset"}} { assert_equal {b 2 c 3} [r zrange zsetc{t} 0 -1 withscores] } + test "ZINTERSTORE with AGGREGATE COUNT - $encoding" { + assert_equal 2 [r zinterstore zsetc{t} 2 zseta{t} zsetb{t} aggregate count] + assert_equal {b 2 c 2} [r zrange zsetc{t} 0 -1 withscores] + } + + test "ZINTERSTORE with AGGREGATE COUNT and WEIGHTS - $encoding" { + assert_equal 2 [r zinterstore zsetc{t} 2 zseta{t} zsetb{t} weights 2 3 aggregate count] + assert_equal {b 5 c 5} [r zrange zsetc{t} 0 -1 withscores] + } + + test "ZUNIONSTORE/ZINTERSTORE with AGGREGATE COUNT - 3 sets - $encoding" { + r del s1{t} s2{t} s3{t} t1{t} + r zadd s1{t} 1 foo 1 bar + r zadd s2{t} 2 foo 2 bar + r zadd s3{t} 3 foo + + assert_equal 1 [r zinterstore t1{t} 3 s1{t} s2{t} s3{t} aggregate count] + assert_equal {foo 3} [r zrange t1{t} 0 -1 withscores] + + assert_equal 2 [r zunionstore t1{t} 3 s1{t} s2{t} s3{t} aggregate count] + assert_equal {bar 2 foo 3} [r zrange t1{t} 0 -1 withscores] + } + + test "ZUNIONSTORE/ZINTERSTORE with AGGREGATE COUNT and WEIGHTS - 3 sets - $encoding" { + assert_equal 1 [r zinterstore t1{t} 3 s1{t} s2{t} s3{t} weights 10 5 3 aggregate count] + assert_equal {foo 18} [r zrange t1{t} 0 -1 withscores] + + assert_equal 2 [r zunionstore t1{t} 3 s1{t} s2{t} s3{t} weights 10 5 3 aggregate count] + assert_equal {bar 15 foo 18} [r zrange t1{t} 0 -1 withscores] + + r del s1{t} s2{t} s3{t} t1{t} + } + foreach cmd {ZUNIONSTORE ZINTERSTORE} { test "$cmd with +inf/-inf scores - $encoding" { r del zsetinf1{t} zsetinf2{t} diff --git a/utils/generate-command-code.py b/utils/generate-command-code.py index 76c8c3b15..8a25039ad 100755 --- a/utils/generate-command-code.py +++ b/utils/generate-command-code.py @@ -34,6 +34,7 @@ GROUPS = { "geo": "COMMAND_GROUP_GEO", "stream": "COMMAND_GROUP_STREAM", "bitmap": "COMMAND_GROUP_BITMAP", + "rate_limit": "COMMAND_GROUP_RATE_LIMIT", } @@ -602,7 +603,8 @@ const char *COMMAND_GROUP_STR[] = { "geo", "stream", "bitmap", - "module" + "module", + "rate_limit" }; const char *commandGroupStr(int index) {