CRC64NVME support (#66)

.github/workflows/ci.yml

@@ -19,8 +19,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [2.5, 2.6, 2.7, jruby]
-    runs-on: ubuntu-22.04 # latest
+        ruby: [2.5, 2.6, 2.7, '3.0', 3.1, 3.2, 3.3, jruby-9.2, jruby-9.3, jruby-9.4]
+    runs-on: ubuntu-latest
     steps:
       - uses: ruby/setup-ruby@v1
         with:
@@ -35,12 +35,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ linux ]
-        ruby: [ 2.7 ]
+        os: [linux]
+        ruby: [ruby]
         arch:
           - x64
           - aarch64
-    runs-on: ubuntu-22.04 # latest
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout Sources
         uses: actions/checkout@v1
@@ -63,7 +63,7 @@ jobs:
     steps:
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
+          ruby-version: ruby
       - uses: actions/checkout@v2
         with:
           submodules: recursive
@@ -78,13 +78,15 @@ jobs:
       fail-fast: false
       matrix:
         arch: [x64, arm64]
-        ruby: [2.5, 2.6, 2.7, jruby]
+        ruby: [2.5, 2.6, 2.7, '3.0', 3.1, 3.2, 3.3, jruby-9.2, jruby-9.3, jruby-9.4]
         exclude:
           # ancient ruby versions not supported on arm64 macos
           - arch: arm64
             ruby: 2.5
           - arch: arm64
             ruby: 2.6
+          - arch: arm64
+            ruby: jruby-9.2
     # Some macos runners are x64, some are arm64, see: https://github.com/actions/runner-images#available-images
     runs-on: ${{ matrix.arch == 'x64' && 'macos-14-large' || 'macos-14' }}
     steps:

.github/workflows/release.yml

@@ -20,11 +20,11 @@ jobs:
       fail-fast: false
       matrix:
         os: [linux]
-        ruby: [2.7]
+        ruby: [ruby]
         arch:
           - x64
           - aarch64
-    runs-on: ubuntu-22.04 # latest
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout Sources
         uses: actions/checkout@v1
@@ -61,7 +61,7 @@ jobs:
     steps:
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
+          ruby-version: ruby
       - name: Generate Gem artifacts
         run: |
           python -c "from urllib.request import urlretrieve; import ssl; ssl._create_default_https_context = ssl._create_unverified_context; urlretrieve('${{ env.BUILDER_HOST }}/releases/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
@@ -79,7 +79,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [macos]
-        ruby: [2.7]
+        ruby: [ruby]
         arch: [x86_64, arm64]
     runs-on: macos-14 # latest
     steps:
@@ -110,7 +110,7 @@ jobs:
     steps:
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
+          ruby-version: ruby
 
       - uses: actions/checkout@v2
         with:
@@ -158,7 +158,7 @@ jobs:
     steps:
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
+          ruby-version: ruby
 
       - uses: actions/checkout@v2
 
@@ -198,7 +198,7 @@ jobs:
     steps:
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
+          ruby-version: ruby
 
       - uses: actions/checkout@v2
 
@@ -218,7 +218,7 @@ jobs:
     steps:
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
+          ruby-version: ruby
       - uses: actions/checkout@v2
 
       - name: Download Release Artifacts
@@ -234,11 +234,11 @@ jobs:
 
   verify-linux-native:
     needs: [package]
-    runs-on: ubuntu-22.04 # latest
+    runs-on: ubuntu-latest
     steps:
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
+          ruby-version: ruby
 
       - uses: actions/checkout@v2
 

.github/workflows/rubocop.yml

@@ -3,16 +3,16 @@ on: [push]
 
 jobs:
   rubocop:
-    runs-on: ubuntu-22.04 # latest
+    runs-on: ubuntu-latest
 
     steps:
       - name: Checkout Sources
         uses: actions/checkout@v2
 
-      - name: Set up Ruby 2.6
+      - name: Setup Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '2.6'
+          ruby-version: ruby
 
       - name: Bundle install
         run: |

README.md

@@ -8,22 +8,15 @@
 * [AWS Common Run Time](https://docs.aws.amazon.com/sdkref/latest/guide/common-runtime.html)
 
 ## Installation 
-AWS CRT bindings are in developer preview and are available from RubyGems as the `aws-crt` gem.  You can install them by adding the `aws-crt`
-gem to your Gemfile.
-
-[Sigv4a](https://docs.aws.amazon.com/general/latest/gr/signing_aws_api_requests.html)
-is an extension to Sigv4 that allows signatures that are valid in more than one region.
-Sigv4a is required to use some services/operations such as
-[S3 Multi-Region Access Points](https://docs.aws.amazon.com/AmazonS3/latest/userguide/MultiRegionAccessPoints.html).
-Currently sigv4a requires the [aws-crt](https://rubygems.org/gems/aws-crt/) gem and a version of the 
-[aws-sigv4](https://rubygems.org/gems/aws-sigv4/) gem built on top of aws-crt (at least version 1.5.0):
+AWS CRT bindings are available from RubyGems as the `aws-crt` gem.
 
 ```ruby
 gem 'aws-sdk-s3', '~> 1'
-gem 'aws-sigv4', '~> 1' # must be at least 1.5.0
-gem 'aws-crt'
+gem 'aws-crt', '~> 0'
 ```
 
+`aws-crt` currently provides fast checksum implementations for CRC32c and CRC64.
+
 ## Versioning
 
 This project uses [semantic versioning](http://semver.org/). You can safely

gems/aws-crt/CHANGELOG.md

@@ -1,12 +1,17 @@
 Unreleased Changes
 ------------------
 
-* Issue - update to the latest aws-crt-ffi.
+* Feature - Support CRC64NVME checksums. 
+
+0.2.1 (2024-06-13)
+------------------
+
+* Issue - Update to the latest aws-crt-ffi.
 
 0.2.0 (2023-11-28)
 ------------------
 
-* Feature - support sigv4-s3express signing.
+* Feature - Support sigv4-s3express signing.
 
 0.1.9 (2023-10-16)
 ------------------
@@ -16,12 +21,13 @@ Unreleased Changes
 0.1.8 (2023-03-28)
 ------------------
 
-* Issue - fix `warning: method redefined` warnings.
+* Issue - Fix `warning: method redefined` warnings.
 
 0.1.7 (2023-02-03)
 ------------------
 
 * Issue - Update to the latest aws-crt-ffi.
+* 
 * Issue - Add support for overriding crt_bin_dir with `AWS_CRT_RUBY_BIN_DIR`.
 
 0.1.6 (2022-08-12)

gems/aws-crt/lib/aws-crt/checksums/crc.rb

@@ -19,6 +19,14 @@ def self.crc32c(str, previous = 0)
           previous
         )
       end
+
+      def self.crc64nvme(str, previous = 0)
+        Aws::Crt::Native.crc64nvme(
+          FFI::MemoryPointer.from_string(str),
+          str.size,
+          previous
+        )
+      end
     end
   end
 end

gems/aws-crt/lib/aws-crt/errors.rb

@@ -52,7 +52,7 @@ def self.error_class(error_name)
       # This requires filtering non-safe characters from the constant
       # name and ensuring it begins with an uppercase letter.
       def self.error_class_constant(error_name)
-        error_name.to_s.gsub(/AWS_ERROR_/, '').split('_').map(&:capitalize).join
+        error_name.to_s.gsub('AWS_ERROR_', '').split('_').map(&:capitalize).join
       end
 
       def self.add_error_constant(constant)

gems/aws-crt/lib/aws-crt/native.rb

@@ -76,7 +76,7 @@ def self.array_to_native(array)
       #   = false)
       # 3. Creates a bang method that does not do automatic error checking.
       def self.attach_function(c_name, params, returns, options = {})
-        ruby_name = c_name.to_s.sub(/aws_crt_/, '').to_sym
+        ruby_name = c_name.to_s.sub('aws_crt_', '').to_sym
         raise_errors = options.fetch(:raise, true)
         options.delete(:raise)
         unless raise_errors
@@ -215,6 +215,7 @@ def self.attach_function(c_name, params, returns, options = {})
       # Checksums
       attach_function :aws_crt_crc32, %i[pointer size_t uint32], :uint32, raise: false
       attach_function :aws_crt_crc32c, %i[pointer size_t uint32], :uint32, raise: false
+      attach_function :aws_crt_crc64nvme, %i[pointer size_t uint64], :uint64, raise: false
 
       # Internal testing API
       attach_function :aws_crt_test_error, [:int], :int

gems/aws-crt/spec/checksums/crc_spec.rb

@@ -43,7 +43,7 @@ def int32_to_base64(num)
       expect(output).to eq(0x91267E8A)
     end
 
-    it 'works with values in one shot' do
+    it 'works with values iterated' do
       output = 0
       32.times do |i|
         output = Aws::Crt::Checksums.crc32([i].pack('C*'), output)
@@ -96,7 +96,7 @@ def int32_to_base64(num)
       expect(output).to eq(0x46DD794E)
     end
 
-    it 'works with values in one shot' do
+    it 'works with values iterated' do
       output = 0
       32.times do |i|
         output = Aws::Crt::Checksums.crc32c([i].pack('C*'), output)
@@ -116,4 +116,51 @@ def int32_to_base64(num)
       skip 'Unable to allocate memory for crc32c huge buffer test'
     end
   end
+
+  describe 'crc64nvme' do
+    test_cases = [
+      { str: '', expected: "AAAAAA==\n" },
+      { str: 'abc', expected: "P8H66w==\n" },
+      { str: 'Hello world', expected: "PzEq2w==\n" }
+    ]
+    test_cases.each do |test_case|
+      it "produces the correct checksum for '#{test_case[:str]}'" do
+        checksum = int32_to_base64(
+          Aws::Crt::Checksums.crc64nvme(test_case[:str])
+        )
+        expect(checksum).to eq(test_case[:expected])
+      end
+    end
+
+    it 'works with zeros in one shot' do
+      output = Aws::Crt::Checksums.crc64nvme(ZERO_CHAR * 32)
+      expect(output).to eq(0xCF3473434D4ECF3B)
+    end
+
+    it 'works with zeros iterated' do
+      output = 0
+      32.times do
+        output = Aws::Crt::Checksums.crc64nvme(ZERO_CHAR, output)
+      end
+      expect(output).to eq(0xCF3473434D4ECF3B)
+    end
+
+    it 'works with values in one shot' do
+      buf = (0...32).to_a.pack('C*')
+      output = Aws::Crt::Checksums.crc64nvme(buf)
+      expect(output).to eq(0xB9D9D4A8492CBD7F)
+    end
+
+    it 'works with a large buffer' do
+      output = Aws::Crt::Checksums.crc64nvme(ZERO_CHAR * 25 * (2**20))
+      expect(output).to eq(0x5B6F5045463CA45E)
+    end
+
+    it 'works with a huge buffer' do
+      output = Aws::Crt::Checksums.crc64nvme(ZERO_CHAR * (INT_MAX + 5))
+      expect(output).to eq(0x2645C28052B1FBB0)
+    rescue NoMemoryError, RangeError
+      skip 'Unable to allocate memory for crc32c huge buffer test'
+    end
+  end
 end