[VOL-5486] Upgrade library versions
Change-Id: I8b4e88699e03f44ee13e467867f45ae3f0a63c4b
Signed-off-by: Abhay Kumar <abhay.kumar@radisys.com>
diff --git a/vendor/github.com/IBM/sarama/.gitignore b/vendor/github.com/IBM/sarama/.gitignore
new file mode 100644
index 0000000..f887957
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/.gitignore
@@ -0,0 +1,31 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+*.test
+
+# Folders
+_obj
+_test
+.vagrant
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+
+/bin
+/coverage.txt
+/profile.out
+/output.json
+
+.idea
diff --git a/vendor/github.com/IBM/sarama/.golangci.yml b/vendor/github.com/IBM/sarama/.golangci.yml
new file mode 100644
index 0000000..520e422
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/.golangci.yml
@@ -0,0 +1,104 @@
+# yaml-language-server: $schema=https://golangci-lint.run/jsonschema/golangci.jsonschema.json
+version: "2"
+linters:
+ default: none
+ enable:
+ - bodyclose
+ - copyloopvar
+ - depguard
+ - dogsled
+ - errcheck
+ - errorlint
+ - funlen
+ - gochecknoinits
+ - gocritic
+ - gocyclo
+ - gosec
+ - govet
+ - misspell
+ - nilerr
+ - unconvert
+ - unused
+ - whitespace
+ settings:
+ depguard:
+ rules:
+ main:
+ deny:
+ - pkg: io/ioutil
+ desc: Use the "io" and "os" packages instead.
+ dupl:
+ threshold: 100
+ funlen:
+ lines: 300
+ statements: 300
+ goconst:
+ min-len: 2
+ min-occurrences: 3
+ gocritic:
+ enabled-checks:
+ - importShadow
+ - nestingReduce
+ - stringsCompare
+ # - unnamedResult
+ # - whyNoLint
+ disabled-checks:
+ - assignOp
+ - appendAssign
+ - commentedOutCode
+ - hugeParam
+ - ifElseChain
+ - singleCaseSwitch
+ - sloppyReassign
+ enabled-tags:
+ - diagnostic
+ - performance
+ # - experimental
+ # - opinionated
+ # - style
+ gocyclo:
+ min-complexity: 99
+ govet:
+ disable:
+ - fieldalignment
+ - shadow
+ enable-all: true
+ misspell:
+ locale: US
+ # exclude some linters from running on certains files.
+ exclusions:
+ generated: lax
+ presets:
+ - comments
+ - common-false-positives
+ - legacy
+ - std-error-handling
+ rules:
+ - linters:
+ - paralleltest
+ path: functional.*_test\.go
+ - path: (.+)\.go$
+ text: 'G115: integer overflow conversion'
+ - path: (.+)\.go$
+ text: 'G404: Use of weak random number generator'
+ paths:
+ - third_party$
+ - builtin$
+ - examples$
+issues:
+ # maximum count of issues with the same text. set to 0 for unlimited. default is 3.
+ max-same-issues: 0
+formatters:
+ enable:
+ - gofmt
+ - goimports
+ settings:
+ goimports:
+ local-prefixes:
+ - github.com/IBM/sarama
+ exclusions:
+ generated: lax
+ paths:
+ - third_party$
+ - builtin$
+ - examples$
diff --git a/vendor/github.com/IBM/sarama/.pre-commit-config.yaml b/vendor/github.com/IBM/sarama/.pre-commit-config.yaml
new file mode 100644
index 0000000..5387686
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/.pre-commit-config.yaml
@@ -0,0 +1,41 @@
+fail_fast: false
+default_install_hook_types: [pre-commit, commit-msg]
+repos:
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v6.0.0
+ hooks:
+ - id: check-merge-conflict
+ - id: check-yaml
+ - id: end-of-file-fixer
+ - id: fix-byte-order-marker
+ - id: mixed-line-ending
+ - id: trailing-whitespace
+ - repo: local
+ hooks:
+ - id: conventional-commit-msg-validation
+ name: commit message conventional validation
+ language: pygrep
+ entry: '^(?:fixup! )?(breaking|build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test){1}(\([\w\-\.]+\))?(!)?: ([\w `])+([\s\S]*)'
+ args: [--multiline, --negate]
+ stages: [commit-msg]
+ - id: commit-msg-needs-to-be-signed-off
+ name: commit message needs to be signed off
+ language: pygrep
+ entry: "^Signed-off-by:"
+ args: [--multiline, --negate]
+ stages: [commit-msg]
+ - id: gofmt
+ name: gofmt
+ description: Format files with gofmt.
+ entry: gofmt -l
+ language: golang
+ files: \.go$
+ args: []
+ - repo: https://github.com/gitleaks/gitleaks
+ rev: v8.28.0
+ hooks:
+ - id: gitleaks
+ - repo: https://github.com/golangci/golangci-lint
+ rev: v2.4.0
+ hooks:
+ - id: golangci-lint
diff --git a/vendor/github.com/IBM/sarama/.whitesource b/vendor/github.com/IBM/sarama/.whitesource
new file mode 100644
index 0000000..26e9c47
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/.whitesource
@@ -0,0 +1,3 @@
+{
+ "settingsInheritedFrom": "ibm-mend-config/mend-config@main"
+}
\ No newline at end of file
diff --git a/vendor/github.com/IBM/sarama/CHANGELOG.md b/vendor/github.com/IBM/sarama/CHANGELOG.md
new file mode 100644
index 0000000..99abeb3
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/CHANGELOG.md
@@ -0,0 +1,1760 @@
+# Changelog
+
+## Version 1.42.2 (2024-02-09)
+
+## What's Changed
+
+⚠️ The go.mod directive has been bumped to 1.18 as the minimum version of Go required for the module. This was necessary to continue to receive updates from some of the third party dependencies that Sarama makes use of for compression.
+
+### :tada: New Features / Improvements
+* feat: update go directive to 1.18 by @dnwe in https://github.com/IBM/sarama/pull/2713
+* feat: return KError instead of errors in AlterConfigs and DescribeConfig by @zhuliquan in https://github.com/IBM/sarama/pull/2472
+### :bug: Fixes
+* fix: don't waste time for backoff on member id required error by @lzakharov in https://github.com/IBM/sarama/pull/2759
+* fix: prevent ConsumerGroup.Close infinitely locking by @maqdev in https://github.com/IBM/sarama/pull/2717
+### :package: Dependency updates
+* chore(deps): bump golang.org/x/net from 0.17.0 to 0.18.0 by @dependabot in https://github.com/IBM/sarama/pull/2716
+* chore(deps): bump golang.org/x/sync to v0.5.0 by @dependabot in https://github.com/IBM/sarama/pull/2718
+* chore(deps): bump github.com/pierrec/lz4/v4 from 4.1.18 to 4.1.19 by @dependabot in https://github.com/IBM/sarama/pull/2739
+* chore(deps): bump golang.org/x/crypto from 0.15.0 to 0.17.0 by @dependabot in https://github.com/IBM/sarama/pull/2748
+* chore(deps): bump the golang-org-x group with 1 update by @dependabot in https://github.com/IBM/sarama/pull/2734
+* chore(deps): bump the golang-org-x group with 2 updates by @dependabot in https://github.com/IBM/sarama/pull/2764
+* chore(deps): bump github.com/pierrec/lz4/v4 from 4.1.19 to 4.1.21 by @dependabot in https://github.com/IBM/sarama/pull/2763
+* chore(deps): bump golang.org/x/crypto from 0.15.0 to 0.17.0 in /examples/exactly_once by @dependabot in https://github.com/IBM/sarama/pull/2749
+* chore(deps): bump golang.org/x/crypto from 0.15.0 to 0.17.0 in /examples/consumergroup by @dependabot in https://github.com/IBM/sarama/pull/2750
+* chore(deps): bump golang.org/x/crypto from 0.15.0 to 0.17.0 in /examples/sasl_scram_client by @dependabot in https://github.com/IBM/sarama/pull/2751
+* chore(deps): bump golang.org/x/crypto from 0.15.0 to 0.17.0 in /examples/interceptors by @dependabot in https://github.com/IBM/sarama/pull/2752
+* chore(deps): bump golang.org/x/crypto from 0.15.0 to 0.17.0 in /examples/http_server by @dependabot in https://github.com/IBM/sarama/pull/2753
+* chore(deps): bump github.com/eapache/go-resiliency from 1.4.0 to 1.5.0 by @dependabot in https://github.com/IBM/sarama/pull/2745
+* chore(deps): bump golang.org/x/crypto from 0.15.0 to 0.17.0 in /examples/txn_producer by @dependabot in https://github.com/IBM/sarama/pull/2754
+* chore(deps): bump go.opentelemetry.io/otel/sdk from 1.19.0 to 1.22.0 in /examples/interceptors by @dependabot in https://github.com/IBM/sarama/pull/2767
+* chore(deps): bump the golang-org-x group with 1 update by @dependabot in https://github.com/IBM/sarama/pull/2793
+* chore(deps): bump go.opentelemetry.io/otel/exporters/stdout/stdoutmetric from 0.42.0 to 1.23.1 in /examples/interceptors by @dependabot in https://github.com/IBM/sarama/pull/2792
+### :wrench: Maintenance
+* fix(examples): housekeeping of code and deps by @dnwe in https://github.com/IBM/sarama/pull/2720
+### :heavy_plus_sign: Other Changes
+* fix(test): retry MockBroker Listen for EADDRINUSE by @dnwe in https://github.com/IBM/sarama/pull/2721
+
+## New Contributors
+* @maqdev made their first contribution in https://github.com/IBM/sarama/pull/2717
+* @zhuliquan made their first contribution in https://github.com/IBM/sarama/pull/2472
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.42.1...v1.42.2
+
+## Version 1.42.1 (2023-11-07)
+
+## What's Changed
+### :bug: Fixes
+* fix: make fetchInitialOffset use correct protocol by @dnwe in https://github.com/IBM/sarama/pull/2705
+* fix(config): relax ClientID validation after 1.0.0 by @dnwe in https://github.com/IBM/sarama/pull/2706
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.42.0...v1.42.1
+
+## Version 1.42.0 (2023-11-02)
+
+## What's Changed
+### :bug: Fixes
+* Asynchronously close brokers during a RefreshBrokers by @bmassemin in https://github.com/IBM/sarama/pull/2693
+* Fix data race on Broker.done channel by @prestona in https://github.com/IBM/sarama/pull/2698
+* fix: data race in Broker.AsyncProduce by @lzakharov in https://github.com/IBM/sarama/pull/2678
+* Fix default retention time value in offset commit by @prestona in https://github.com/IBM/sarama/pull/2700
+* fix(txmgr): ErrOffsetsLoadInProgress is retriable by @dnwe in https://github.com/IBM/sarama/pull/2701
+### :wrench: Maintenance
+* chore(ci): improve ossf scorecard result by @dnwe in https://github.com/IBM/sarama/pull/2685
+* chore(ci): add kafka 3.6.0 to FVT and versions by @dnwe in https://github.com/IBM/sarama/pull/2692
+### :heavy_plus_sign: Other Changes
+* chore(ci): ossf scorecard.yml by @dnwe in https://github.com/IBM/sarama/pull/2683
+* fix(ci): always run CodeQL on every commit by @dnwe in https://github.com/IBM/sarama/pull/2689
+* chore(doc): add OpenSSF Scorecard badge by @dnwe in https://github.com/IBM/sarama/pull/2691
+
+## New Contributors
+* @bmassemin made their first contribution in https://github.com/IBM/sarama/pull/2693
+* @lzakharov made their first contribution in https://github.com/IBM/sarama/pull/2678
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.41.3...v1.42.0
+
+## Version 1.41.3 (2023-10-17)
+
+## What's Changed
+### :bug: Fixes
+* fix: pre-compile regex for parsing kafka version by @qshuai in https://github.com/IBM/sarama/pull/2663
+* fix(client): ignore empty Metadata responses when refreshing by @HaoSunUber in https://github.com/IBM/sarama/pull/2672
+### :package: Dependency updates
+* chore(deps): bump the golang-org-x group with 2 updates by @dependabot in https://github.com/IBM/sarama/pull/2661
+* chore(deps): bump golang.org/x/net from 0.16.0 to 0.17.0 by @dependabot in https://github.com/IBM/sarama/pull/2671
+### :memo: Documentation
+* fix(docs): correct topic name in rebalancing strategy example by @maksadbek in https://github.com/IBM/sarama/pull/2657
+
+## New Contributors
+* @maksadbek made their first contribution in https://github.com/IBM/sarama/pull/2657
+* @qshuai made their first contribution in https://github.com/IBM/sarama/pull/2663
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.41.2...v1.41.3
+
+## Version 1.41.2 (2023-09-12)
+
+## What's Changed
+### :tada: New Features / Improvements
+* perf: Alloc records in batch by @ronanh in https://github.com/IBM/sarama/pull/2646
+### :bug: Fixes
+* fix(consumer): guard against nil client by @dnwe in https://github.com/IBM/sarama/pull/2636
+* fix(consumer): don't retry session if ctx canceled by @dnwe in https://github.com/IBM/sarama/pull/2642
+* fix: use least loaded broker to refresh metadata by @HaoSunUber in https://github.com/IBM/sarama/pull/2645
+### :package: Dependency updates
+* chore(deps): bump the golang-org-x group with 1 update by @dependabot in https://github.com/IBM/sarama/pull/2641
+
+## New Contributors
+* @HaoSunUber made their first contribution in https://github.com/IBM/sarama/pull/2645
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.41.1...v1.41.2
+
+## Version 1.41.1 (2023-08-30)
+
+## What's Changed
+### :bug: Fixes
+* fix(proto): handle V3 member metadata and empty owned partitions by @dnwe in https://github.com/IBM/sarama/pull/2618
+* fix: make clear that error is configuration issue not server error by @hindessm in https://github.com/IBM/sarama/pull/2628
+* fix(client): force Event Hubs to use V1_0_0_0 by @dnwe in https://github.com/IBM/sarama/pull/2633
+* fix: add retries to alter user scram creds by @hindessm in https://github.com/IBM/sarama/pull/2632
+### :wrench: Maintenance
+* chore(lint): bump golangci-lint and tweak config by @dnwe in https://github.com/IBM/sarama/pull/2620
+### :memo: Documentation
+* fix(doc): add missing doc for mock consumer by @hsweif in https://github.com/IBM/sarama/pull/2386
+* chore(proto): doc CreateTopics/JoinGroup fields by @dnwe in https://github.com/IBM/sarama/pull/2627
+### :heavy_plus_sign: Other Changes
+* chore(gh): add new style issue templates by @dnwe in https://github.com/IBM/sarama/pull/2624
+
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.41.0...v1.41.1
+
+## Version 1.41.0 (2023-08-21)
+
+## What's Changed
+### :rotating_light: Breaking Changes
+
+Note: this version of Sarama has had a big overhaul in its adherence to the use of the right Kafka protocol versions for the given Config Version. It has also bumped the default Version set in Config (where one is not supplied) to 2.1.0. This is in preparation for Kafka 4.0 dropping support for protocol versions older than 2.1. If you are using Sarama against Kafka clusters older than v2.1.0, or using it against Azure EventHubs then you will likely have to change your application code to pin to the appropriate Version.
+
+* chore(config): make DefaultVersion V2_0_0_0 by @dnwe in https://github.com/IBM/sarama/pull/2572
+* chore(config): make DefaultVersion V2_1_0_0 by @dnwe in https://github.com/IBM/sarama/pull/2574
+### :tada: New Features / Improvements
+* Implement resolve_canonical_bootstrap_servers_only by @gebn in https://github.com/IBM/sarama/pull/2156
+* feat: sleep when throttled (KIP-219) by @hindessm in https://github.com/IBM/sarama/pull/2536
+* feat: add isValidVersion to protocol types by @dnwe in https://github.com/IBM/sarama/pull/2538
+* fix(consumer): use newer LeaveGroup as appropriate by @dnwe in https://github.com/IBM/sarama/pull/2544
+* Add support for up to version 4 List Groups API by @prestona in https://github.com/IBM/sarama/pull/2541
+* fix(producer): use newer ProduceReq as appropriate by @dnwe in https://github.com/IBM/sarama/pull/2546
+* fix(proto): ensure req+resp requiredVersion match by @dnwe in https://github.com/IBM/sarama/pull/2548
+* chore(proto): permit CreatePartitionsRequest V1 by @dnwe in https://github.com/IBM/sarama/pull/2549
+* chore(proto): permit AlterConfigsRequest V1 by @dnwe in https://github.com/IBM/sarama/pull/2550
+* chore(proto): permit DeleteGroupsRequest V1 by @dnwe in https://github.com/IBM/sarama/pull/2551
+* fix(proto): correct JoinGroup usage for wider version range by @dnwe in https://github.com/IBM/sarama/pull/2553
+* fix(consumer): use full range of FetchRequest vers by @dnwe in https://github.com/IBM/sarama/pull/2554
+* fix(proto): use range of OffsetCommitRequest vers by @dnwe in https://github.com/IBM/sarama/pull/2555
+* fix(proto): use full range of MetadataRequest by @dnwe in https://github.com/IBM/sarama/pull/2556
+* fix(proto): use fuller ranges of supported proto by @dnwe in https://github.com/IBM/sarama/pull/2558
+* fix(proto): use full range of SyncGroupRequest by @dnwe in https://github.com/IBM/sarama/pull/2565
+* fix(proto): use full range of ListGroupsRequest by @dnwe in https://github.com/IBM/sarama/pull/2568
+* feat(proto): support for Metadata V6-V10 by @dnwe in https://github.com/IBM/sarama/pull/2566
+* fix(proto): use full ranges for remaining proto by @dnwe in https://github.com/IBM/sarama/pull/2570
+* feat(proto): add remaining protocol for V2.1 by @dnwe in https://github.com/IBM/sarama/pull/2573
+* feat: add new error for MockDeleteTopicsResponse by @javiercri in https://github.com/IBM/sarama/pull/2475
+* feat(gzip): switch to klauspost/compress gzip by @dnwe in https://github.com/IBM/sarama/pull/2600
+### :bug: Fixes
+* fix: correct unsupported version check by @hindessm in https://github.com/IBM/sarama/pull/2528
+* fix: avoiding burning cpu if all partitions are paused by @napallday in https://github.com/IBM/sarama/pull/2532
+* extend throttling metric scope by @hindessm in https://github.com/IBM/sarama/pull/2533
+* Fix printing of final metrics by @prestona in https://github.com/IBM/sarama/pull/2545
+* fix(consumer): cannot automatically fetch newly-added partitions unless restart by @napallday in https://github.com/IBM/sarama/pull/2563
+* bug: implement unsigned modulus for partitioning with crc32 hashing by @csm8118 in https://github.com/IBM/sarama/pull/2560
+* fix: avoid logging value of proxy.Dialer by @prestona in https://github.com/IBM/sarama/pull/2569
+* fix(test): add missing closes to admin client tests by @dnwe in https://github.com/IBM/sarama/pull/2594
+* fix(test): ensure some more clients are closed by @dnwe in https://github.com/IBM/sarama/pull/2595
+* fix(examples): sync exactly_once and consumergroup by @dnwe in https://github.com/IBM/sarama/pull/2614
+* fix(fvt): fresh metrics registry for each test by @dnwe in https://github.com/IBM/sarama/pull/2616
+* fix(test): flaky test TestFuncOffsetManager by @napallday in https://github.com/IBM/sarama/pull/2609
+### :package: Dependency updates
+* chore(deps): bump the golang-org-x group with 1 update by @dependabot in https://github.com/IBM/sarama/pull/2542
+* chore(deps): bump the golang-org-x group with 1 update by @dependabot in https://github.com/IBM/sarama/pull/2561
+* chore(deps): bump module github.com/pierrec/lz4/v4 to v4.1.18 by @dnwe in https://github.com/IBM/sarama/pull/2589
+* chore(deps): bump module github.com/jcmturner/gokrb5/v8 to v8.4.4 by @dnwe in https://github.com/IBM/sarama/pull/2587
+* chore(deps): bump github.com/eapache/go-xerial-snappy digest to c322873 by @dnwe in https://github.com/IBM/sarama/pull/2586
+* chore(deps): bump module github.com/klauspost/compress to v1.16.7 by @dnwe in https://github.com/IBM/sarama/pull/2588
+* chore(deps): bump github.com/eapache/go-resiliency from 1.3.0 to 1.4.0 by @dependabot in https://github.com/IBM/sarama/pull/2598
+### :wrench: Maintenance
+* fix(fvt): ensure fully-replicated at test start by @hindessm in https://github.com/IBM/sarama/pull/2531
+* chore: rollup fvt kafka to latest three by @dnwe in https://github.com/IBM/sarama/pull/2537
+* Merge the two CONTRIBUTING.md's by @prestona in https://github.com/IBM/sarama/pull/2543
+* fix(test): test timing error by @hindessm in https://github.com/IBM/sarama/pull/2552
+* chore(ci): tidyup and improve actions workflows by @dnwe in https://github.com/IBM/sarama/pull/2557
+* fix(test): shutdown MockBroker by @dnwe in https://github.com/IBM/sarama/pull/2571
+* chore(proto): match HeartbeatResponse version by @dnwe in https://github.com/IBM/sarama/pull/2576
+* chore(test): ensure MockBroker closed within test by @dnwe in https://github.com/IBM/sarama/pull/2575
+* chore(test): ensure all mockresponses use version by @dnwe in https://github.com/IBM/sarama/pull/2578
+* chore(ci): use latest Go in actions by @dnwe in https://github.com/IBM/sarama/pull/2580
+* chore(test): speedup some slow tests by @dnwe in https://github.com/IBM/sarama/pull/2579
+* chore(test): use modern protocol versions in FVT by @dnwe in https://github.com/IBM/sarama/pull/2581
+* chore(test): fix a couple of leaks by @dnwe in https://github.com/IBM/sarama/pull/2591
+* feat(fvt): experiment with per-kafka-version image by @dnwe in https://github.com/IBM/sarama/pull/2592
+* chore(ci): replace toxiproxy client dep by @dnwe in https://github.com/IBM/sarama/pull/2593
+* feat(fvt): add healthcheck, depends_on and --wait by @dnwe in https://github.com/IBM/sarama/pull/2601
+* fix(fvt): handle msgset vs batchset by @dnwe in https://github.com/IBM/sarama/pull/2603
+* fix(fvt): Metadata version in ensureFullyReplicated by @dnwe in https://github.com/IBM/sarama/pull/2612
+* fix(fvt): versioned cfg for invalid topic producer by @dnwe in https://github.com/IBM/sarama/pull/2613
+* chore(fvt): tweak to work across more versions by @dnwe in https://github.com/IBM/sarama/pull/2615
+* feat(fvt): test wider range of kafkas by @dnwe in https://github.com/IBM/sarama/pull/2605
+### :memo: Documentation
+* fix(example): check if msg channel is closed by @ioanzicu in https://github.com/IBM/sarama/pull/2479
+* chore: use go install for installing sarama tools by @vigith in https://github.com/IBM/sarama/pull/2599
+
+## New Contributors
+* @gebn made their first contribution in https://github.com/IBM/sarama/pull/2156
+* @prestona made their first contribution in https://github.com/IBM/sarama/pull/2543
+* @ioanzicu made their first contribution in https://github.com/IBM/sarama/pull/2479
+* @csm8118 made their first contribution in https://github.com/IBM/sarama/pull/2560
+* @javiercri made their first contribution in https://github.com/IBM/sarama/pull/2475
+* @vigith made their first contribution in https://github.com/IBM/sarama/pull/2599
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.40.1...v1.41.0
+
+## Version 1.40.1 (2023-07-27)
+
+## What's Changed
+### :tada: New Features / Improvements
+* Use buffer pools for decompression by @ronanh in https://github.com/IBM/sarama/pull/2484
+* feat: support for Kerberos authentication with a credentials cache. by @mrogaski in https://github.com/IBM/sarama/pull/2457
+### :bug: Fixes
+* Fix some retry issues by @hindessm in https://github.com/IBM/sarama/pull/2517
+* fix: admin retry logic by @hindessm in https://github.com/IBM/sarama/pull/2519
+* Add some retry logic to more admin client functions by @hindessm in https://github.com/IBM/sarama/pull/2520
+* fix: concurrent issue on updateMetadataMs by @napallday in https://github.com/IBM/sarama/pull/2522
+* fix(test): allow testing of skipped test without IsTransactional panic by @hindessm in https://github.com/IBM/sarama/pull/2525
+### :package: Dependency updates
+* chore(deps): bump the golang-org-x group with 2 updates by @dependabot in https://github.com/IBM/sarama/pull/2509
+* chore(deps): bump github.com/klauspost/compress from 1.15.14 to 1.16.6 by @dependabot in https://github.com/IBM/sarama/pull/2513
+* chore(deps): bump github.com/stretchr/testify from 1.8.1 to 1.8.3 by @dependabot in https://github.com/IBM/sarama/pull/2512
+### :wrench: Maintenance
+* chore(ci): migrate probot-stale to actions/stale by @dnwe in https://github.com/IBM/sarama/pull/2496
+* chore(ci): bump golangci version, cleanup, depguard config by @EladLeev in https://github.com/IBM/sarama/pull/2504
+* Clean up some typos and docs/help mistakes by @hindessm in https://github.com/IBM/sarama/pull/2514
+### :heavy_plus_sign: Other Changes
+* chore(ci): add simple apidiff workflow by @dnwe in https://github.com/IBM/sarama/pull/2497
+* chore(ci): bump actions/setup-go from 3 to 4 by @dependabot in https://github.com/IBM/sarama/pull/2508
+* fix(comments): PauseAll and ResumeAll by @napallday in https://github.com/IBM/sarama/pull/2523
+
+## New Contributors
+* @EladLeev made their first contribution in https://github.com/IBM/sarama/pull/2504
+* @hindessm made their first contribution in https://github.com/IBM/sarama/pull/2514
+* @ronanh made their first contribution in https://github.com/IBM/sarama/pull/2484
+* @mrogaski made their first contribution in https://github.com/IBM/sarama/pull/2457
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.40.0...v1.40.1
+
+## Version 1.40.0 (2023-07-17)
+
+## What's Changed
+
+Note: this is the first release after the transition of Sarama ownership from Shopify to IBM in https://github.com/IBM/sarama/issues/2461
+
+### :rotating_light: Breaking Changes
+
+- chore: migrate module to github.com/IBM/sarama by @dnwe in https://github.com/IBM/sarama/pull/2492
+- fix: restore (\*OffsetCommitRequest) AddBlock func by @dnwe in https://github.com/IBM/sarama/pull/2494
+
+### :bug: Fixes
+
+- fix(consumer): don't retry FindCoordinator forever by @dnwe in https://github.com/IBM/sarama/pull/2427
+- fix(metrics): fix race condition when calling Broker.Open() twice by @vincentbernat in https://github.com/IBM/sarama/pull/2428
+- fix: use version 4 of DescribeGroupsRequest only if kafka broker vers… …ion is >= 2.4 by @faillefer in https://github.com/IBM/sarama/pull/2451
+- Fix HighWaterMarkOffset of mocks partition consumer by @gr8web in https://github.com/IBM/sarama/pull/2447
+- fix: prevent data race in balance strategy by @napallday in https://github.com/IBM/sarama/pull/2453
+
+### :package: Dependency updates
+
+- chore(deps): bump golang.org/x/net from 0.5.0 to 0.7.0 by @dependabot in https://github.com/IBM/sarama/pull/2452
+
+### :wrench: Maintenance
+
+- chore: add kafka 3.3.2 by @dnwe in https://github.com/IBM/sarama/pull/2434
+- chore(ci): remove Shopify/shopify-cla-action by @dnwe in https://github.com/IBM/sarama/pull/2489
+- chore: bytes.Equal instead bytes.Compare by @testwill in https://github.com/IBM/sarama/pull/2485
+
+## New Contributors
+
+- @dependabot made their first contribution in https://github.com/IBM/sarama/pull/2452
+- @gr8web made their first contribution in https://github.com/IBM/sarama/pull/2447
+- @testwill made their first contribution in https://github.com/IBM/sarama/pull/2485
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.38.1...v1.40.0
+
+## Version 1.38.1 (2023-01-22)
+
+## What's Changed
+### :bug: Fixes
+* fix(example): correct `records-number` param in txn producer readme by @diallo-han in https://github.com/IBM/sarama/pull/2420
+* fix: use newConsumer method in newConsumerGroup method by @Lumotheninja in https://github.com/IBM/sarama/pull/2424
+### :package: Dependency updates
+* chore(deps): bump module github.com/klauspost/compress to v1.15.14 by @dnwe in https://github.com/IBM/sarama/pull/2410
+* chore(deps): bump module golang.org/x/net to v0.5.0 by @dnwe in https://github.com/IBM/sarama/pull/2413
+* chore(deps): bump module github.com/stretchr/testify to v1.8.1 by @dnwe in https://github.com/IBM/sarama/pull/2411
+* chore(deps): bump module github.com/xdg-go/scram to v1.1.2 by @dnwe in https://github.com/IBM/sarama/pull/2412
+* chore(deps): bump module golang.org/x/sync to v0.1.0 by @dnwe in https://github.com/IBM/sarama/pull/2414
+* chore(deps): bump github.com/eapache/go-xerial-snappy digest to bf00bc1 by @dnwe in https://github.com/IBM/sarama/pull/2418
+
+## New Contributors
+* @diallo-han made their first contribution in https://github.com/IBM/sarama/pull/2420
+* @Lumotheninja made their first contribution in https://github.com/IBM/sarama/pull/2424
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.38.0...v1.38.1
+
+## Version 1.38.0 (2023-01-08)
+
+## What's Changed
+### :tada: New Features / Improvements
+* feat(producer): improve memory usage of zstd encoder by using our own pool management by @rtreffer in https://github.com/IBM/sarama/pull/2375
+* feat(proto): implement and use MetadataRequest v7 by @dnwe in https://github.com/IBM/sarama/pull/2388
+* feat(metrics): add protocol-requests-rate metric by @auntan in https://github.com/IBM/sarama/pull/2373
+### :bug: Fixes
+* fix(proto): track and supply leader epoch to FetchRequest by @dnwe in https://github.com/IBM/sarama/pull/2389
+* fix(example): improve arg name used for tls skip verify by @michaeljmarshall in https://github.com/IBM/sarama/pull/2385
+* fix(zstd): default back to GOMAXPROCS concurrency by @bgreenlee in https://github.com/IBM/sarama/pull/2404
+* fix(producer): add nil check while producer is retrying by @hsweif in https://github.com/IBM/sarama/pull/2387
+* fix(producer): return errors for every message in retryBatch to avoid producer hang forever by @cch123 in https://github.com/IBM/sarama/pull/2378
+* fix(metrics): fix race when accessing metric registry by @vincentbernat in https://github.com/IBM/sarama/pull/2409
+### :package: Dependency updates
+* chore(deps): bump golang.org/x/net to v0.4.0 by @dnwe in https://github.com/IBM/sarama/pull/2403
+### :wrench: Maintenance
+* chore(ci): replace set-output command in GH Action by @dnwe in https://github.com/IBM/sarama/pull/2390
+* chore(ci): include kafka 3.3.1 in testing matrix by @dnwe in https://github.com/IBM/sarama/pull/2406
+
+## New Contributors
+* @michaeljmarshall made their first contribution in https://github.com/IBM/sarama/pull/2385
+* @bgreenlee made their first contribution in https://github.com/IBM/sarama/pull/2404
+* @hsweif made their first contribution in https://github.com/IBM/sarama/pull/2387
+* @cch123 made their first contribution in https://github.com/IBM/sarama/pull/2378
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.37.2...v1.38.0
+
+## Version 1.37.2 (2022-10-04)
+
+## What's Changed
+### :bug: Fixes
+* fix: ensure updateMetaDataMs is 64-bit aligned by @dnwe in https://github.com/IBM/sarama/pull/2356
+### :heavy_plus_sign: Other Changes
+* fix: bump go.mod specification to go 1.17 by @dnwe in https://github.com/IBM/sarama/pull/2357
+
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.37.1...v1.37.2
+
+## Version 1.37.1 (2022-10-04)
+
+## What's Changed
+### :bug: Fixes
+* fix: support existing deprecated Rebalance.Strategy field usage by @spongecaptain in https://github.com/IBM/sarama/pull/2352
+* fix(test): consumer group rebalance strategy compatibility by @Jacob-bzx in https://github.com/IBM/sarama/pull/2353
+* fix(producer): replace time.After with time.Timer to avoid high memory usage by @Jacob-bzx in https://github.com/IBM/sarama/pull/2355
+
+## New Contributors
+* @spongecaptain made their first contribution in https://github.com/IBM/sarama/pull/2352
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.37.0...v1.37.1
+
+## Version 1.37.0 (2022-09-28)
+
+## What's Changed
+
+### :rotating_light: Breaking Changes
+* Due to a change in [github.com/klauspost/compress v1.15.10](https://github.com/klauspost/compress/releases/tag/v1.15.10), Sarama v1.37.0 requires Go 1.17 going forward, unfortunately due to an oversight this wasn't reflected in the go.mod declaration at time of release.
+
+### :tada: New Features / Improvements
+* feat(consumer): support multiple balance strategies by @Jacob-bzx in https://github.com/IBM/sarama/pull/2339
+* feat(producer): transactional API by @ryarnyah in https://github.com/IBM/sarama/pull/2295
+* feat(mocks): support key in MockFetchResponse. by @Skandalik in https://github.com/IBM/sarama/pull/2328
+### :bug: Fixes
+* fix: avoid panic when Metadata.RefreshFrequency is 0 by @Jacob-bzx in https://github.com/IBM/sarama/pull/2329
+* fix(consumer): avoid pushing unrelated responses to paused children by @pkoutsovasilis in https://github.com/IBM/sarama/pull/2317
+* fix: prevent metrics leak with cleanup by @auntan in https://github.com/IBM/sarama/pull/2340
+* fix: race condition(may panic) when closing consumer group by @Jacob-bzx in https://github.com/IBM/sarama/pull/2331
+* fix(consumer): default ResetInvalidOffsets to true by @dnwe in https://github.com/IBM/sarama/pull/2345
+* Validate the `Config` when creating a mock producer/consumer by @joewreschnig in https://github.com/IBM/sarama/pull/2327
+### :package: Dependency updates
+* chore(deps): bump module github.com/pierrec/lz4/v4 to v4.1.16 by @dnwe in https://github.com/IBM/sarama/pull/2335
+* chore(deps): bump golang.org/x/net digest to bea034e by @dnwe in https://github.com/IBM/sarama/pull/2333
+* chore(deps): bump golang.org/x/sync digest to 7f9b162 by @dnwe in https://github.com/IBM/sarama/pull/2334
+* chore(deps): bump golang.org/x/net digest to f486391 by @dnwe in https://github.com/IBM/sarama/pull/2348
+* chore(deps): bump module github.com/shopify/toxiproxy/v2 to v2.5.0 by @dnwe in https://github.com/IBM/sarama/pull/2336
+* chore(deps): bump module github.com/klauspost/compress to v1.15.11 by @dnwe in https://github.com/IBM/sarama/pull/2349
+* chore(deps): bump module github.com/pierrec/lz4/v4 to v4.1.17 by @dnwe in https://github.com/IBM/sarama/pull/2350
+### :wrench: Maintenance
+* chore(ci): bump kafka-versions to latest by @dnwe in https://github.com/IBM/sarama/pull/2346
+* chore(ci): bump go-versions to N and N-1 by @dnwe in https://github.com/IBM/sarama/pull/2347
+
+## New Contributors
+* @Jacob-bzx made their first contribution in https://github.com/IBM/sarama/pull/2329
+* @pkoutsovasilis made their first contribution in https://github.com/IBM/sarama/pull/2317
+* @Skandalik made their first contribution in https://github.com/IBM/sarama/pull/2328
+* @auntan made their first contribution in https://github.com/IBM/sarama/pull/2340
+* @ryarnyah made their first contribution in https://github.com/IBM/sarama/pull/2295
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.36.0...v1.37.0
+
+## Version 1.36.0 (2022-08-11)
+
+## What's Changed
+### :tada: New Features / Improvements
+* feat: add option to propagate OffsetOutOfRange error by @dkolistratova in https://github.com/IBM/sarama/pull/2252
+* feat(producer): expose ProducerMessage.byteSize() function by @k8scat in https://github.com/IBM/sarama/pull/2315
+* feat(metrics): track consumer fetch request rates by @dnwe in https://github.com/IBM/sarama/pull/2299
+### :bug: Fixes
+* fix(consumer): avoid submitting empty fetch requests when paused by @raulnegreiros in https://github.com/IBM/sarama/pull/2143
+### :package: Dependency updates
+* chore(deps): bump module github.com/klauspost/compress to v1.15.9 by @dnwe in https://github.com/IBM/sarama/pull/2304
+* chore(deps): bump golang.org/x/net digest to c7608f3 by @dnwe in https://github.com/IBM/sarama/pull/2301
+* chore(deps): bump golangci/golangci-lint-action action to v3 by @dnwe in https://github.com/IBM/sarama/pull/2311
+* chore(deps): bump golang.org/x/net digest to 07c6da5 by @dnwe in https://github.com/IBM/sarama/pull/2307
+* chore(deps): bump github actions versions (major) by @dnwe in https://github.com/IBM/sarama/pull/2313
+* chore(deps): bump module github.com/jcmturner/gofork to v1.7.6 by @dnwe in https://github.com/IBM/sarama/pull/2305
+* chore(deps): bump golang.org/x/sync digest to 886fb93 by @dnwe in https://github.com/IBM/sarama/pull/2302
+* chore(deps): bump module github.com/jcmturner/gokrb5/v8 to v8.4.3 by @dnwe in https://github.com/IBM/sarama/pull/2303
+### :wrench: Maintenance
+* chore: add kafka 3.1.1 to the version matrix by @dnwe in https://github.com/IBM/sarama/pull/2300
+### :heavy_plus_sign: Other Changes
+* Migrate off probot-CLA to new GitHub Action by @cursedcoder in https://github.com/IBM/sarama/pull/2294
+* Forgot to remove cla probot by @cursedcoder in https://github.com/IBM/sarama/pull/2297
+* chore(lint): re-enable a small amount of go-critic by @dnwe in https://github.com/IBM/sarama/pull/2312
+
+## New Contributors
+* @cursedcoder made their first contribution in https://github.com/IBM/sarama/pull/2294
+* @dkolistratova made their first contribution in https://github.com/IBM/sarama/pull/2252
+* @k8scat made their first contribution in https://github.com/IBM/sarama/pull/2315
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.35.0...v1.36.0
+
+## Version 1.35.0 (2022-07-22)
+
+## What's Changed
+### :bug: Fixes
+* fix: fix metadata retry backoff invalid when get metadata failed by @Stephan14 in https://github.com/IBM/sarama/pull/2256
+* fix(balance): sort and de-deplicate memberIDs by @dnwe in https://github.com/IBM/sarama/pull/2285
+* fix: prevent DescribeLogDirs hang in admin client by @zerowidth in https://github.com/IBM/sarama/pull/2269
+* fix: include assignment-less members in SyncGroup by @dnwe in https://github.com/IBM/sarama/pull/2292
+### :package: Dependency updates
+* chore(deps): bump module github.com/stretchr/testify to v1.8.0 by @dnwe in https://github.com/IBM/sarama/pull/2284
+* chore(deps): bump module github.com/eapache/go-resiliency to v1.3.0 by @dnwe in https://github.com/IBM/sarama/pull/2283
+* chore(deps): bump golang.org/x/net digest to 1185a90 by @dnwe in https://github.com/IBM/sarama/pull/2279
+* chore(deps): bump module github.com/pierrec/lz4/v4 to v4.1.15 by @dnwe in https://github.com/IBM/sarama/pull/2281
+* chore(deps): bump module github.com/klauspost/compress to v1.15.8 by @dnwe in https://github.com/IBM/sarama/pull/2280
+### :wrench: Maintenance
+* chore: rename `any` func to avoid identifier by @dnwe in https://github.com/IBM/sarama/pull/2272
+* chore: add and test against kafka 3.2.0 by @dnwe in https://github.com/IBM/sarama/pull/2288
+* chore: document Fetch protocol fields by @dnwe in https://github.com/IBM/sarama/pull/2289
+### :heavy_plus_sign: Other Changes
+* chore(ci): fix redirect with GITHUB_STEP_SUMMARY by @dnwe in https://github.com/IBM/sarama/pull/2286
+* fix(test): permit ECONNRESET in TestInitProducerID by @dnwe in https://github.com/IBM/sarama/pull/2287
+* fix: ensure empty or devel version valid by @dnwe in https://github.com/IBM/sarama/pull/2291
+
+## New Contributors
+* @zerowidth made their first contribution in https://github.com/IBM/sarama/pull/2269
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.34.1...v1.35.0
+
+## Version 1.34.1 (2022-06-07)
+
+## What's Changed
+### :bug: Fixes
+* fix(examples): check session.Context().Done() in examples/consumergroup by @zxc111 in https://github.com/IBM/sarama/pull/2240
+* fix(protocol): move AuthorizedOperations into GroupDescription of DescribeGroupsResponse by @aiquestion in https://github.com/IBM/sarama/pull/2247
+* fix(protocol): tidyup DescribeGroupsResponse by @dnwe in https://github.com/IBM/sarama/pull/2248
+* fix(consumer): range balance strategy not like reference by @njhartwell in https://github.com/IBM/sarama/pull/2245
+### :wrench: Maintenance
+* chore(ci): experiment with using tparse by @dnwe in https://github.com/IBM/sarama/pull/2236
+* chore(deps): bump thirdparty dependencies to latest releases by @dnwe in https://github.com/IBM/sarama/pull/2242
+
+## New Contributors
+* @zxc111 made their first contribution in https://github.com/IBM/sarama/pull/2240
+* @njhartwell made their first contribution in https://github.com/IBM/sarama/pull/2245
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.34.0...v1.34.1
+
+## Version 1.34.0 (2022-05-30)
+
+## What's Changed
+### :tada: New Features / Improvements
+* KIP-345: support static membership by @aiquestion in https://github.com/IBM/sarama/pull/2230
+### :bug: Fixes
+* fix: KIP-368 use receiver goroutine to process all sasl v1 responses by @k-wall in https://github.com/IBM/sarama/pull/2234
+### :wrench: Maintenance
+* chore(deps): bump module github.com/pierrec/lz4 to v4 by @dnwe in https://github.com/IBM/sarama/pull/2231
+* chore(deps): bump golang.org/x/net digest to 2e3eb7b by @dnwe in https://github.com/IBM/sarama/pull/2232
+
+## New Contributors
+* @aiquestion made their first contribution in https://github.com/IBM/sarama/pull/2230
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.33.0...v1.34.0
+
+## Version 1.33.0 (2022-05-11)
+
+## What's Changed
+### :rotating_light: Breaking Changes
+
+**Note: with this change, the user of Sarama is required to use Go 1.13's errors.Is etc (rather then ==) when forming conditionals returned by this library.**
+* feat: make `ErrOutOfBrokers` wrap the underlying error that prevented connections to the brokers by @k-wall in https://github.com/IBM/sarama/pull/2131
+
+
+### :tada: New Features / Improvements
+* feat(message): add UnmarshalText method to CompressionCodec by @vincentbernat in https://github.com/IBM/sarama/pull/2172
+* KIP-368 : Allow SASL Connections to Periodically Re-Authenticate by @k-wall in https://github.com/IBM/sarama/pull/2197
+* feat: add batched CreateACLs func to ClusterAdmin by @nkostoulas in https://github.com/IBM/sarama/pull/2191
+### :bug: Fixes
+* fix: TestRecordBatchDecoding failing sporadically by @k-wall in https://github.com/IBM/sarama/pull/2154
+* feat(test): add an fvt for broker deadlock by @dnwe in https://github.com/IBM/sarama/pull/2144
+* fix: avoid starvation in subscriptionManager by @dnwe in https://github.com/IBM/sarama/pull/2109
+* fix: remove "Is your cluster reachable?" from msg by @dnwe in https://github.com/IBM/sarama/pull/2165
+* fix: remove trailing fullstop from error strings by @dnwe in https://github.com/IBM/sarama/pull/2166
+* fix: return underlying sasl error message by @dnwe in https://github.com/IBM/sarama/pull/2164
+* fix: potential data race on a global variable by @pior in https://github.com/IBM/sarama/pull/2171
+* fix: AdminClient | CreateACLs | check for error in response, return error if needed by @omris94 in https://github.com/IBM/sarama/pull/2185
+* producer: ensure that the management message (fin) is never "leaked" by @niamster in https://github.com/IBM/sarama/pull/2182
+* fix: prevent RefreshBrokers leaking old brokers by @k-wall in https://github.com/IBM/sarama/pull/2203
+* fix: prevent RefreshController leaking controller by @k-wall in https://github.com/IBM/sarama/pull/2204
+* fix: prevent AsyncProducer retryBatch from leaking by @k-wall in https://github.com/IBM/sarama/pull/2208
+* fix: prevent metrics leak when authenticate fails by @Stephan14 in https://github.com/IBM/sarama/pull/2205
+* fix: prevent deadlock between subscription manager and consumer goroutines by @niamster in https://github.com/IBM/sarama/pull/2194
+* fix: prevent idempotent producer epoch exhaustion by @ladislavmacoun in https://github.com/IBM/sarama/pull/2178
+* fix(test): mockbroker offsetResponse vers behavior by @dnwe in https://github.com/IBM/sarama/pull/2213
+* fix: cope with OffsetsLoadInProgress on Join+Sync by @dnwe in https://github.com/IBM/sarama/pull/2214
+* fix: make default MaxWaitTime 500ms by @dnwe in https://github.com/IBM/sarama/pull/2227
+### :package: Dependency updates
+* chore(deps): bump xdg-go/scram and klauspost/compress by @dnwe in https://github.com/IBM/sarama/pull/2170
+### :wrench: Maintenance
+* fix(test): skip TestReadOnlyAndAllCommittedMessages by @dnwe in https://github.com/IBM/sarama/pull/2161
+* fix(test): remove t.Parallel() by @dnwe in https://github.com/IBM/sarama/pull/2162
+* chore(ci): bump along to Go 1.17+1.18 and bump golangci-lint by @dnwe in https://github.com/IBM/sarama/pull/2183
+* chore: switch to multi-arch compatible docker images by @dnwe in https://github.com/IBM/sarama/pull/2210
+### :heavy_plus_sign: Other Changes
+* Remediate a number go-routine leaks (mainly test issues) by @k-wall in https://github.com/IBM/sarama/pull/2198
+* chore: retract v1.32.0 due to #2150 by @dnwe in https://github.com/IBM/sarama/pull/2199
+* chore: bump functional test timeout to 12m by @dnwe in https://github.com/IBM/sarama/pull/2200
+* fix(admin): make DeleteRecords err consistent by @dnwe in https://github.com/IBM/sarama/pull/2226
+
+## New Contributors
+* @k-wall made their first contribution in https://github.com/IBM/sarama/pull/2154
+* @pior made their first contribution in https://github.com/IBM/sarama/pull/2171
+* @omris94 made their first contribution in https://github.com/IBM/sarama/pull/2185
+* @vincentbernat made their first contribution in https://github.com/IBM/sarama/pull/2172
+* @niamster made their first contribution in https://github.com/IBM/sarama/pull/2182
+* @ladislavmacoun made their first contribution in https://github.com/IBM/sarama/pull/2178
+* @nkostoulas made their first contribution in https://github.com/IBM/sarama/pull/2191
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.32.0...v1.33.0
+
+## Version 1.32.0 (2022-02-24)
+
+### ⚠️ This release has been superseded by v1.33.0 and should _not_ be used.
+
+* chore: retract v1.32.0 due to #2150 by @dnwe in https://github.com/IBM/sarama/pull/2199
+
+---
+
+## What's Changed
+### :bug: Fixes
+* Fix deadlock when closing Broker in brokerProducer by @slaunay in https://github.com/IBM/sarama/pull/2133
+### :package: Dependency updates
+* chore: refresh dependencies to latest by @dnwe in https://github.com/IBM/sarama/pull/2159
+### :wrench: Maintenance
+* fix: rework RebalancingMultiplePartitions test by @dnwe in https://github.com/IBM/sarama/pull/2130
+* fix(test): use Sarama transactional producer by @dnwe in https://github.com/IBM/sarama/pull/1939
+* chore: enable t.Parallel() wherever possible by @dnwe in https://github.com/IBM/sarama/pull/2138
+### :heavy_plus_sign: Other Changes
+* chore: restrict to 1 testbinary at once by @dnwe in https://github.com/IBM/sarama/pull/2145
+* chore: restrict to 1 parallel test at once by @dnwe in https://github.com/IBM/sarama/pull/2146
+* Remove myself from codeowners by @bai in https://github.com/IBM/sarama/pull/2147
+* chore: add retractions for known bad versions by @dnwe in https://github.com/IBM/sarama/pull/2160
+
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.31.1...v1.32.0
+
+## Version 1.31.1 (2022-02-01)
+
+- #2126 - @bai - Populate missing kafka versions
+- #2124 - @bai - Add Kafka 3.1.0 to CI matrix, migrate to bitnami kafka image
+- #2123 - @bai - Update klauspost/compress to 0.14
+- #2122 - @dnwe - fix(test): make it simpler to re-use toxiproxy
+- #2119 - @bai - Add Kafka 3.1.0 version number
+- #2005 - @raulnegreiros - feat: add methods to pause/resume consumer's consumption
+- #2051 - @seveas - Expose the TLS connection state of a broker connection
+- #2117 - @wuhuizuo - feat: add method MockApiVersionsResponse.SetApiKeys
+- #2110 - @dnwe - fix: ensure heartbeats only stop after cleanup
+- #2113 - @mosceo - Fix typo
+
+## Version 1.31.0 (2022-01-18)
+
+## What's Changed
+### :tada: New Features / Improvements
+* feat: expose IncrementalAlterConfigs API in admin.go by @fengyinqiao in https://github.com/IBM/sarama/pull/2088
+* feat: allow AsyncProducer to have MaxOpenRequests inflight produce requests per broker by @xujianhai666 in https://github.com/IBM/sarama/pull/1686
+* Support request pipelining in AsyncProducer by @slaunay in https://github.com/IBM/sarama/pull/2094
+### :bug: Fixes
+* fix(test): add fluent interface for mocks where missing by @grongor in https://github.com/IBM/sarama/pull/2080
+* fix(test): test for ConsumePartition with OffsetOldest by @grongor in https://github.com/IBM/sarama/pull/2081
+* fix: set HWMO during creation of partitionConsumer (fix incorrect HWMO before first fetch) by @grongor in https://github.com/IBM/sarama/pull/2082
+* fix: ignore non-nil but empty error strings in Describe/Alter client quotas responses by @agriffaut in https://github.com/IBM/sarama/pull/2096
+* fix: skip over KIP-482 tagged fields by @dnwe in https://github.com/IBM/sarama/pull/2107
+* fix: clear preferredReadReplica if broker shutdown by @dnwe in https://github.com/IBM/sarama/pull/2108
+* fix(test): correct wrong offsets in mock Consumer by @grongor in https://github.com/IBM/sarama/pull/2078
+* fix: correct bugs in DescribeGroupsResponse by @dnwe in https://github.com/IBM/sarama/pull/2111
+### :wrench: Maintenance
+* chore: bump runtime and test dependencies by @dnwe in https://github.com/IBM/sarama/pull/2100
+### :memo: Documentation
+* docs: refresh README.md for Kafka 3.0.0 by @dnwe in https://github.com/IBM/sarama/pull/2099
+### :heavy_plus_sign: Other Changes
+* Fix typo by @mosceo in https://github.com/IBM/sarama/pull/2084
+
+## New Contributors
+* @grongor made their first contribution in https://github.com/IBM/sarama/pull/2080
+* @fengyinqiao made their first contribution in https://github.com/IBM/sarama/pull/2088
+* @xujianhai666 made their first contribution in https://github.com/IBM/sarama/pull/1686
+* @mosceo made their first contribution in https://github.com/IBM/sarama/pull/2084
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.30.1...v1.31.0
+
+## Version 1.30.1 (2021-12-04)
+
+## What's Changed
+### :tada: New Features / Improvements
+* feat(zstd): pass level param through to compress/zstd encoder by @lizthegrey in https://github.com/IBM/sarama/pull/2045
+### :bug: Fixes
+* fix: set min-go-version to 1.16 by @troyanov in https://github.com/IBM/sarama/pull/2048
+* logger: fix debug logs' formatting directives by @utrack in https://github.com/IBM/sarama/pull/2054
+* fix: stuck on the batch with zero records length by @pachmu in https://github.com/IBM/sarama/pull/2057
+* fix: only update preferredReadReplica if valid by @dnwe in https://github.com/IBM/sarama/pull/2076
+### :wrench: Maintenance
+* chore: add release notes configuration by @dnwe in https://github.com/IBM/sarama/pull/2046
+* chore: confluent platform version bump by @lizthegrey in https://github.com/IBM/sarama/pull/2070
+
+## Notes
+* ℹ️ from Sarama 1.30.x onward the minimum version of Go toolchain required is 1.16.x
+
+## New Contributors
+* @troyanov made their first contribution in https://github.com/IBM/sarama/pull/2048
+* @lizthegrey made their first contribution in https://github.com/IBM/sarama/pull/2045
+* @utrack made their first contribution in https://github.com/IBM/sarama/pull/2054
+* @pachmu made their first contribution in https://github.com/IBM/sarama/pull/2057
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.30.0...v1.30.1
+
+## Version 1.30.0 (2021-09-29)
+
+⚠️ This release has been superseded by v1.30.1 and should _not_ be used.
+
+**regression**: enabling rackawareness causes severe throughput drops (#2071) — fixed in v1.30.1 via #2076
+
+---
+
+ℹ️ **Note: from Sarama 1.30.0 the minimum version of Go toolchain required is 1.16.x**
+
+---
+
+# New Features / Improvements
+
+- #1983 - @zifengyu - allow configure AllowAutoTopicCreation argument in metadata refresh
+- #2000 - @matzew - Using xdg-go module for SCRAM
+- #2003 - @gdm85 - feat: add counter metrics for consumer group join/sync and their failures
+- #1992 - @zhaomoran - feat: support SaslHandshakeRequest v0 for SCRAM
+- #2006 - @faillefer - Add support for DeleteOffsets operation
+- #1909 - @agriffaut - KIP-546 Client quota APIs
+- #1633 - @aldelucca1 - feat: allow balance strategies to provide initial state
+- #1275 - @dnwe - log: add a DebugLogger that proxies to Logger
+- #2018 - @dnwe - feat: use DebugLogger reference for goldenpath log
+- #2019 - @dnwe - feat: add logging & a metric for producer throttle
+- #2023 - @dnwe - feat: add Controller() to ClusterAdmin interface
+- #2025 - @dnwe - feat: support ApiVersionsRequest V3 protocol
+- #2028 - @dnwe - feat: send ApiVersionsRequest on broker open
+- #2034 - @bai - Add support for kafka 3.0.0
+
+# Fixes
+
+- #1990 - @doxsch - fix: correctly pass ValidateOnly through to CreatePartitionsRequest
+- #1988 - @LubergAlexander - fix: correct WithCustomFallbackPartitioner implementation
+- #2001 - @HurSungYun - docs: inform AsyncProducer Close pitfalls
+- #1973 - @qiangmzsx - fix: metrics still taking up too much memory when metrics.UseNilMetrics=true
+- #2007 - @bai - Add support for Go 1.17
+- #2009 - @dnwe - fix: enable nilerr linter and fix iferr checks
+- #2010 - @dnwe - chore: enable exportloopref and misspell linters
+- #2013 - @faillefer - fix(test): disable encoded response/request check when map contains multiple elements
+- #2015 - @bai - Change default branch to main
+- #1718 - @crivera-fastly - fix: correct the error handling in client.InitProducerID()
+- #1984 - @null-sleep - fix(test): bump confluentPlatformVersion from 6.1.1 to 6.2.0
+- #2016 - @dnwe - chore: replace deprecated Go calls
+- #2017 - @dnwe - chore: delete legacy vagrant script
+- #2020 - @dnwe - fix(test): remove testLogger from TrackLeader test
+- #2024 - @dnwe - chore: bump toxiproxy container to v2.1.5
+- #2033 - @bai - Update dependencies
+- #2031 - @gdm85 - docs: do not mention buffered messages in sync producer Close method
+- #2035 - @dnwe - chore: populate the missing kafka versions
+- #2038 - @dnwe - feat: add a fuzzing workflow to github actions
+
+## New Contributors
+* @zifengyu made their first contribution in https://github.com/IBM/sarama/pull/1983
+* @doxsch made their first contribution in https://github.com/IBM/sarama/pull/1990
+* @LubergAlexander made their first contribution in https://github.com/IBM/sarama/pull/1988
+* @HurSungYun made their first contribution in https://github.com/IBM/sarama/pull/2001
+* @gdm85 made their first contribution in https://github.com/IBM/sarama/pull/2003
+* @qiangmzsx made their first contribution in https://github.com/IBM/sarama/pull/1973
+* @zhaomoran made their first contribution in https://github.com/IBM/sarama/pull/1992
+* @faillefer made their first contribution in https://github.com/IBM/sarama/pull/2006
+* @crivera-fastly made their first contribution in https://github.com/IBM/sarama/pull/1718
+* @null-sleep made their first contribution in https://github.com/IBM/sarama/pull/1984
+
+**Full Changelog**: https://github.com/IBM/sarama/compare/v1.29.1...v1.30.0
+
+## Version 1.29.1 (2021-06-24)
+
+# New Features / Improvements
+
+- #1966 - @ajanikow - KIP-339: Add Incremental Config updates API
+- #1964 - @ajanikow - Add DelegationToken ResourceType
+
+# Fixes
+
+- #1962 - @hanxiaolin - fix(consumer): call interceptors when MaxProcessingTime expire
+- #1971 - @KerryJava - fix kafka-producer-performance throughput panic
+- #1968 - @dnwe - chore: bump golang.org/x versions
+- #1956 - @joewreschnig - Allow checking the entire `ProducerMessage` in the mock producers
+- #1963 - @dnwe - fix: ensure backoff timer is re-used
+- #1949 - @dnwe - fix: explicitly use uint64 for payload length
+
+## Version 1.29.0 (2021-05-07)
+
+### New Features / Improvements
+
+- #1917 - @arkady-emelyanov - KIP-554: Add Broker-side SCRAM Config API
+- #1869 - @wyndhblb - zstd: encode+decode performance improvements
+- #1541 - @izolight - add String, (Un)MarshalText for acl types.
+- #1921 - @bai - Add support for Kafka 2.8.0
+
+### Fixes
+- #1936 - @dnwe - fix(consumer): follow preferred broker
+- #1933 - @ozzieba - Use gofork for encoding/asn1 to fix ASN errors during Kerberos authentication
+- #1929 - @celrenheit - Handle isolation level in Offset(Request|Response) and require stable offset in FetchOffset(Request|Response)
+- #1926 - @dnwe - fix: correct initial CodeQL findings
+- #1925 - @bai - Test out CodeQL
+- #1923 - @bestgopher - Remove redundant switch-case, fix doc typos
+- #1922 - @bai - Update go dependencies
+- #1898 - @mmaslankaprv - Parsing only known control batches value
+- #1887 - @withshubh - Fix: issues affecting code quality
+
+## Version 1.28.0 (2021-02-15)
+
+**Note that with this release we change `RoundRobinBalancer` strategy to match Java client behavior. See #1788 for details.**
+
+- #1870 - @kvch - Update Kerberos library to latest major
+- #1876 - @bai - Update docs, reference pkg.go.dev
+- #1846 - @wclaeys - Do not ignore Consumer.Offsets.AutoCommit.Enable config on Close
+- #1747 - @XSAM - fix: mock sync producer does not handle the offset while sending messages
+- #1863 - @bai - Add support for Kafka 2.7.0 + update lz4 and klauspost/compress dependencies
+- #1788 - @kzinglzy - feat[balance_strategy]: announcing a new round robin balance strategy
+- #1862 - @bai - Fix CI setenv permissions issues
+- #1832 - @ilyakaznacheev - Update Godoc link to pkg.go.dev
+- #1822 - @danp - KIP-392: Allow consumers to fetch from closest replica
+
+## Version 1.27.2 (2020-10-21)
+
+### Improvements
+
+#1750 - @krantideep95 Adds missing mock responses for mocking consumer group
+
+## Fixes
+
+#1817 - reverts #1785 - Add private method to Client interface to prevent implementation
+
+## Version 1.27.1 (2020-10-07)
+
+### Improvements
+
+#1775 - @d1egoaz - Adds a Producer Interceptor example
+#1781 - @justin-chen - Refresh brokers given list of seed brokers
+#1784 - @justin-chen - Add randomize seed broker method
+#1790 - @d1egoaz - remove example binary
+#1798 - @bai - Test against Go 1.15
+#1785 - @justin-chen - Add private method to Client interface to prevent implementation
+#1802 - @uvw - Support Go 1.13 error unwrapping
+
+## Fixes
+
+#1791 - @stanislavkozlovski - bump default version to 1.0.0
+
+## Version 1.27.0 (2020-08-11)
+
+### Improvements
+
+#1466 - @rubenvp8510 - Expose kerberos fast negotiation configuration
+#1695 - @KJTsanaktsidis - Use docker-compose to run the functional tests
+#1699 - @wclaeys - Consumer group support for manually comitting offsets
+#1714 - @bai - Bump Go to version 1.14.3, golangci-lint to 1.27.0
+#1726 - @d1egoaz - Include zstd on the functional tests
+#1730 - @d1egoaz - KIP-42 Add producer and consumer interceptors
+#1738 - @varun06 - fixed variable names that are named same as some std lib package names
+#1741 - @varun06 - updated zstd dependency to latest v1.10.10
+#1743 - @varun06 - Fixed declaration dependencies and other lint issues in code base
+#1763 - @alrs - remove deprecated tls options from test
+#1769 - @bai - Add support for Kafka 2.6.0
+
+## Fixes
+
+#1697 - @kvch - Use gofork for encoding/asn1 to fix ASN errors during Kerberos authentication
+#1744 - @alrs - Fix isBalanced Function Signature
+
+## Version 1.26.4 (2020-05-19)
+
+## Fixes
+
+- #1701 - @d1egoaz - Set server name only for the current broker
+- #1694 - @dnwe - testfix: set KAFKA_HEAP_OPTS for zk and kafka
+
+## Version 1.26.3 (2020-05-07)
+
+## Fixes
+
+- #1692 - @d1egoaz - Set tls ServerName to fix issue: either ServerName or InsecureSkipVerify must be specified in the tls.Config
+
+## Version 1.26.2 (2020-05-06)
+
+## ⚠️ Known Issues
+
+This release has been marked as not ready for production and may be unstable, please use v1.26.4.
+
+### Improvements
+
+- #1560 - @iyacontrol - add sync pool for gzip 1-9
+- #1605 - @dnwe - feat: protocol support for V11 fetch w/ rackID
+- #1617 - @sladkoff / @dwi-di / @random-dwi - Add support for alter/list partition reassignements APIs
+- #1632 - @bai - Add support for Go 1.14
+- #1640 - @random-dwi - Feature/fix list partition reassignments
+- #1646 - @mimaison - Add DescribeLogDirs to admin client
+- #1667 - @bai - Add support for kafka 2.5.0
+
+## Fixes
+
+- #1594 - @sladkoff - Sets ConfigEntry.Default flag in addition to the ConfigEntry.Source for Kafka versions > V1_1_0_0
+- #1601 - @alrs - fix: remove use of testing.T.FailNow() inside goroutine
+- #1602 - @d1egoaz - adds a note about consumer groups Consume method
+- #1607 - @darklore - Fix memory leak when Broker.Open and Broker.Close called repeatedly
+- #1613 - @wblakecaldwell - Updated "retrying" log message when BackoffFunc implemented
+- #1614 - @alrs - produce_response.go: Remove Unused Functions
+- #1619 - @alrs - tools/kafka-producer-performance: prune unused flag variables
+- #1639 - @agriffaut - Handle errors with no message but error code
+- #1643 - @kzinglzy - fix `config.net.keepalive`
+- #1644 - @KJTsanaktsidis - Fix brokers continually allocating new Session IDs
+- #1645 - @Stephan14 - Remove broker(s) which no longer exist in metadata
+- #1650 - @lavoiesl - Return the response error in heartbeatLoop
+- #1661 - @KJTsanaktsidis - Fix "broker received out of order sequence" when brokers die
+- #1666 - @KevinJCross - Bugfix: Allow TLS connections to work over socks proxy.
+
+## Version 1.26.1 (2020-02-04)
+
+Improvements:
+- Add requests-in-flight metric ([1539](https://github.com/IBM/sarama/pull/1539))
+- Fix misleading example for cluster admin ([1595](https://github.com/IBM/sarama/pull/1595))
+- Replace Travis with GitHub Actions, linters housekeeping ([1573](https://github.com/IBM/sarama/pull/1573))
+- Allow BalanceStrategy to provide custom assignment data ([1592](https://github.com/IBM/sarama/pull/1592))
+
+Bug Fixes:
+- Adds back Consumer.Offsets.CommitInterval to fix API ([1590](https://github.com/IBM/sarama/pull/1590))
+- Fix error message s/CommitInterval/AutoCommit.Interval ([1589](https://github.com/IBM/sarama/pull/1589))
+
+## Version 1.26.0 (2020-01-24)
+
+New Features:
+- Enable zstd compression
+ ([1574](https://github.com/IBM/sarama/pull/1574),
+ [1582](https://github.com/IBM/sarama/pull/1582))
+- Support headers in tools kafka-console-producer
+ ([1549](https://github.com/IBM/sarama/pull/1549))
+
+Improvements:
+- Add SASL AuthIdentity to SASL frames (authzid)
+ ([1585](https://github.com/IBM/sarama/pull/1585)).
+
+Bug Fixes:
+- Sending messages with ZStd compression enabled fails in multiple ways
+ ([1252](https://github.com/IBM/sarama/issues/1252)).
+- Use the broker for any admin on BrokerConfig
+ ([1571](https://github.com/IBM/sarama/pull/1571)).
+- Set DescribeConfigRequest Version field
+ ([1576](https://github.com/IBM/sarama/pull/1576)).
+- ConsumerGroup flooding logs with client/metadata update req
+ ([1578](https://github.com/IBM/sarama/pull/1578)).
+- MetadataRequest version in DescribeCluster
+ ([1580](https://github.com/IBM/sarama/pull/1580)).
+- Fix deadlock in consumer group handleError
+ ([1581](https://github.com/IBM/sarama/pull/1581))
+- Fill in the Fetch{Request,Response} protocol
+ ([1582](https://github.com/IBM/sarama/pull/1582)).
+- Retry topic request on ControllerNotAvailable
+ ([1586](https://github.com/IBM/sarama/pull/1586)).
+
+## Version 1.25.0 (2020-01-13)
+
+New Features:
+- Support TLS protocol in kafka-producer-performance
+ ([1538](https://github.com/IBM/sarama/pull/1538)).
+- Add support for kafka 2.4.0
+ ([1552](https://github.com/IBM/sarama/pull/1552)).
+
+Improvements:
+- Allow the Consumer to disable auto-commit offsets
+ ([1164](https://github.com/IBM/sarama/pull/1164)).
+- Produce records with consistent timestamps
+ ([1455](https://github.com/IBM/sarama/pull/1455)).
+
+Bug Fixes:
+- Fix incorrect SetTopicMetadata name mentions
+ ([1534](https://github.com/IBM/sarama/pull/1534)).
+- Fix client.tryRefreshMetadata Println
+ ([1535](https://github.com/IBM/sarama/pull/1535)).
+- Fix panic on calling updateMetadata on closed client
+ ([1531](https://github.com/IBM/sarama/pull/1531)).
+- Fix possible faulty metrics in TestFuncProducing
+ ([1545](https://github.com/IBM/sarama/pull/1545)).
+
+## Version 1.24.1 (2019-10-31)
+
+New Features:
+- Add DescribeLogDirs Request/Response pair
+ ([1520](https://github.com/IBM/sarama/pull/1520)).
+
+Bug Fixes:
+- Fix ClusterAdmin returning invalid controller ID on DescribeCluster
+ ([1518](https://github.com/IBM/sarama/pull/1518)).
+- Fix issue with consumergroup not rebalancing when new partition is added
+ ([1525](https://github.com/IBM/sarama/pull/1525)).
+- Ensure consistent use of read/write deadlines
+ ([1529](https://github.com/IBM/sarama/pull/1529)).
+
+## Version 1.24.0 (2019-10-09)
+
+New Features:
+- Add sticky partition assignor
+ ([1416](https://github.com/IBM/sarama/pull/1416)).
+- Switch from cgo zstd package to pure Go implementation
+ ([1477](https://github.com/IBM/sarama/pull/1477)).
+
+Improvements:
+- Allow creating ClusterAdmin from client
+ ([1415](https://github.com/IBM/sarama/pull/1415)).
+- Set KafkaVersion in ListAcls method
+ ([1452](https://github.com/IBM/sarama/pull/1452)).
+- Set request version in CreateACL ClusterAdmin method
+ ([1458](https://github.com/IBM/sarama/pull/1458)).
+- Set request version in DeleteACL ClusterAdmin method
+ ([1461](https://github.com/IBM/sarama/pull/1461)).
+- Handle missed error codes on TopicMetaDataRequest and GroupCoordinatorRequest
+ ([1464](https://github.com/IBM/sarama/pull/1464)).
+- Remove direct usage of gofork
+ ([1465](https://github.com/IBM/sarama/pull/1465)).
+- Add support for Go 1.13
+ ([1478](https://github.com/IBM/sarama/pull/1478)).
+- Improve behavior of NewMockListAclsResponse
+ ([1481](https://github.com/IBM/sarama/pull/1481)).
+
+Bug Fixes:
+- Fix race condition in consumergroup example
+ ([1434](https://github.com/IBM/sarama/pull/1434)).
+- Fix brokerProducer goroutine leak
+ ([1442](https://github.com/IBM/sarama/pull/1442)).
+- Use released version of lz4 library
+ ([1469](https://github.com/IBM/sarama/pull/1469)).
+- Set correct version in MockDeleteTopicsResponse
+ ([1484](https://github.com/IBM/sarama/pull/1484)).
+- Fix CLI help message typo
+ ([1494](https://github.com/IBM/sarama/pull/1494)).
+
+Known Issues:
+- Please **don't** use Zstd, as it doesn't work right now.
+ See https://github.com/IBM/sarama/issues/1252
+
+## Version 1.23.1 (2019-07-22)
+
+Bug Fixes:
+- Fix fetch delete bug record
+ ([1425](https://github.com/IBM/sarama/pull/1425)).
+- Handle SASL/OAUTHBEARER token rejection
+ ([1428](https://github.com/IBM/sarama/pull/1428)).
+
+## Version 1.23.0 (2019-07-02)
+
+New Features:
+- Add support for Kafka 2.3.0
+ ([1418](https://github.com/IBM/sarama/pull/1418)).
+- Add support for ListConsumerGroupOffsets v2
+ ([1374](https://github.com/IBM/sarama/pull/1374)).
+- Add support for DeleteConsumerGroup
+ ([1417](https://github.com/IBM/sarama/pull/1417)).
+- Add support for SASLVersion configuration
+ ([1410](https://github.com/IBM/sarama/pull/1410)).
+- Add kerberos support
+ ([1366](https://github.com/IBM/sarama/pull/1366)).
+
+Improvements:
+- Improve sasl_scram_client example
+ ([1406](https://github.com/IBM/sarama/pull/1406)).
+- Fix shutdown and race-condition in consumer-group example
+ ([1404](https://github.com/IBM/sarama/pull/1404)).
+- Add support for error codes 77—81
+ ([1397](https://github.com/IBM/sarama/pull/1397)).
+- Pool internal objects allocated per message
+ ([1385](https://github.com/IBM/sarama/pull/1385)).
+- Reduce packet decoder allocations
+ ([1373](https://github.com/IBM/sarama/pull/1373)).
+- Support timeout when fetching metadata
+ ([1359](https://github.com/IBM/sarama/pull/1359)).
+
+Bug Fixes:
+- Fix fetch size integer overflow
+ ([1376](https://github.com/IBM/sarama/pull/1376)).
+- Handle and log throttled FetchResponses
+ ([1383](https://github.com/IBM/sarama/pull/1383)).
+- Refactor misspelled word Resouce to Resource
+ ([1368](https://github.com/IBM/sarama/pull/1368)).
+
+## Version 1.22.1 (2019-04-29)
+
+Improvements:
+- Use zstd 1.3.8
+ ([1350](https://github.com/IBM/sarama/pull/1350)).
+- Add support for SaslHandshakeRequest v1
+ ([1354](https://github.com/IBM/sarama/pull/1354)).
+
+Bug Fixes:
+- Fix V5 MetadataRequest nullable topics array
+ ([1353](https://github.com/IBM/sarama/pull/1353)).
+- Use a different SCRAM client for each broker connection
+ ([1349](https://github.com/IBM/sarama/pull/1349)).
+- Fix AllowAutoTopicCreation for MetadataRequest greater than v3
+ ([1344](https://github.com/IBM/sarama/pull/1344)).
+
+## Version 1.22.0 (2019-04-09)
+
+New Features:
+- Add Offline Replicas Operation to Client
+ ([1318](https://github.com/IBM/sarama/pull/1318)).
+- Allow using proxy when connecting to broker
+ ([1326](https://github.com/IBM/sarama/pull/1326)).
+- Implement ReadCommitted
+ ([1307](https://github.com/IBM/sarama/pull/1307)).
+- Add support for Kafka 2.2.0
+ ([1331](https://github.com/IBM/sarama/pull/1331)).
+- Add SASL SCRAM-SHA-512 and SCRAM-SHA-256 mechanismes
+ ([1331](https://github.com/IBM/sarama/pull/1295)).
+
+Improvements:
+- Unregister all broker metrics on broker stop
+ ([1232](https://github.com/IBM/sarama/pull/1232)).
+- Add SCRAM authentication example
+ ([1303](https://github.com/IBM/sarama/pull/1303)).
+- Add consumergroup examples
+ ([1304](https://github.com/IBM/sarama/pull/1304)).
+- Expose consumer batch size metric
+ ([1296](https://github.com/IBM/sarama/pull/1296)).
+- Add TLS options to console producer and consumer
+ ([1300](https://github.com/IBM/sarama/pull/1300)).
+- Reduce client close bookkeeping
+ ([1297](https://github.com/IBM/sarama/pull/1297)).
+- Satisfy error interface in create responses
+ ([1154](https://github.com/IBM/sarama/pull/1154)).
+- Please lint gods
+ ([1346](https://github.com/IBM/sarama/pull/1346)).
+
+Bug Fixes:
+- Fix multi consumer group instance crash
+ ([1338](https://github.com/IBM/sarama/pull/1338)).
+- Update lz4 to latest version
+ ([1347](https://github.com/IBM/sarama/pull/1347)).
+- Retry ErrNotCoordinatorForConsumer in new consumergroup session
+ ([1231](https://github.com/IBM/sarama/pull/1231)).
+- Fix cleanup error handler
+ ([1332](https://github.com/IBM/sarama/pull/1332)).
+- Fix rate condition in PartitionConsumer
+ ([1156](https://github.com/IBM/sarama/pull/1156)).
+
+## Version 1.21.0 (2019-02-24)
+
+New Features:
+- Add CreateAclRequest, DescribeAclRequest, DeleteAclRequest
+ ([1236](https://github.com/IBM/sarama/pull/1236)).
+- Add DescribeTopic, DescribeConsumerGroup, ListConsumerGroups, ListConsumerGroupOffsets admin requests
+ ([1178](https://github.com/IBM/sarama/pull/1178)).
+- Implement SASL/OAUTHBEARER
+ ([1240](https://github.com/IBM/sarama/pull/1240)).
+
+Improvements:
+- Add Go mod support
+ ([1282](https://github.com/IBM/sarama/pull/1282)).
+- Add error codes 73—76
+ ([1239](https://github.com/IBM/sarama/pull/1239)).
+- Add retry backoff function
+ ([1160](https://github.com/IBM/sarama/pull/1160)).
+- Maintain metadata in the producer even when retries are disabled
+ ([1189](https://github.com/IBM/sarama/pull/1189)).
+- Include ReplicaAssignment in ListTopics
+ ([1274](https://github.com/IBM/sarama/pull/1274)).
+- Add producer performance tool
+ ([1222](https://github.com/IBM/sarama/pull/1222)).
+- Add support LogAppend timestamps
+ ([1258](https://github.com/IBM/sarama/pull/1258)).
+
+Bug Fixes:
+- Fix potential deadlock when a heartbeat request fails
+ ([1286](https://github.com/IBM/sarama/pull/1286)).
+- Fix consuming compacted topic
+ ([1227](https://github.com/IBM/sarama/pull/1227)).
+- Set correct Kafka version for DescribeConfigsRequest v1
+ ([1277](https://github.com/IBM/sarama/pull/1277)).
+- Update kafka test version
+ ([1273](https://github.com/IBM/sarama/pull/1273)).
+
+## Version 1.20.1 (2019-01-10)
+
+New Features:
+- Add optional replica id in offset request
+ ([1100](https://github.com/IBM/sarama/pull/1100)).
+
+Improvements:
+- Implement DescribeConfigs Request + Response v1 & v2
+ ([1230](https://github.com/IBM/sarama/pull/1230)).
+- Reuse compression objects
+ ([1185](https://github.com/IBM/sarama/pull/1185)).
+- Switch from png to svg for GoDoc link in README
+ ([1243](https://github.com/IBM/sarama/pull/1243)).
+- Fix typo in deprecation notice for FetchResponseBlock.Records
+ ([1242](https://github.com/IBM/sarama/pull/1242)).
+- Fix typos in consumer metadata response file
+ ([1244](https://github.com/IBM/sarama/pull/1244)).
+
+Bug Fixes:
+- Revert to individual msg retries for non-idempotent
+ ([1203](https://github.com/IBM/sarama/pull/1203)).
+- Respect MaxMessageBytes limit for uncompressed messages
+ ([1141](https://github.com/IBM/sarama/pull/1141)).
+
+## Version 1.20.0 (2018-12-10)
+
+New Features:
+ - Add support for zstd compression
+ ([#1170](https://github.com/IBM/sarama/pull/1170)).
+ - Add support for Idempotent Producer
+ ([#1152](https://github.com/IBM/sarama/pull/1152)).
+ - Add support support for Kafka 2.1.0
+ ([#1229](https://github.com/IBM/sarama/pull/1229)).
+ - Add support support for OffsetCommit request/response pairs versions v1 to v5
+ ([#1201](https://github.com/IBM/sarama/pull/1201)).
+ - Add support support for OffsetFetch request/response pair up to version v5
+ ([#1198](https://github.com/IBM/sarama/pull/1198)).
+
+Improvements:
+ - Export broker's Rack setting
+ ([#1173](https://github.com/IBM/sarama/pull/1173)).
+ - Always use latest patch version of Go on CI
+ ([#1202](https://github.com/IBM/sarama/pull/1202)).
+ - Add error codes 61 to 72
+ ([#1195](https://github.com/IBM/sarama/pull/1195)).
+
+Bug Fixes:
+ - Fix build without cgo
+ ([#1182](https://github.com/IBM/sarama/pull/1182)).
+ - Fix go vet suggestion in consumer group file
+ ([#1209](https://github.com/IBM/sarama/pull/1209)).
+ - Fix typos in code and comments
+ ([#1228](https://github.com/IBM/sarama/pull/1228)).
+
+## Version 1.19.0 (2018-09-27)
+
+New Features:
+ - Implement a higher-level consumer group
+ ([#1099](https://github.com/IBM/sarama/pull/1099)).
+
+Improvements:
+ - Add support for Go 1.11
+ ([#1176](https://github.com/IBM/sarama/pull/1176)).
+
+Bug Fixes:
+ - Fix encoding of `MetadataResponse` with version 2 and higher
+ ([#1174](https://github.com/IBM/sarama/pull/1174)).
+ - Fix race condition in mock async producer
+ ([#1174](https://github.com/IBM/sarama/pull/1174)).
+
+## Version 1.18.0 (2018-09-07)
+
+New Features:
+ - Make `Partitioner.RequiresConsistency` vary per-message
+ ([#1112](https://github.com/IBM/sarama/pull/1112)).
+ - Add customizable partitioner
+ ([#1118](https://github.com/IBM/sarama/pull/1118)).
+ - Add `ClusterAdmin` support for `CreateTopic`, `DeleteTopic`, `CreatePartitions`,
+ `DeleteRecords`, `DescribeConfig`, `AlterConfig`, `CreateACL`, `ListAcls`, `DeleteACL`
+ ([#1055](https://github.com/IBM/sarama/pull/1055)).
+
+Improvements:
+ - Add support for Kafka 2.0.0
+ ([#1149](https://github.com/IBM/sarama/pull/1149)).
+ - Allow setting `LocalAddr` when dialing an address to support multi-homed hosts
+ ([#1123](https://github.com/IBM/sarama/pull/1123)).
+ - Simpler offset management
+ ([#1127](https://github.com/IBM/sarama/pull/1127)).
+
+Bug Fixes:
+ - Fix mutation of `ProducerMessage.MetaData` when producing to Kafka
+ ([#1110](https://github.com/IBM/sarama/pull/1110)).
+ - Fix consumer block when response did not contain all the
+ expected topic/partition blocks
+ ([#1086](https://github.com/IBM/sarama/pull/1086)).
+ - Fix consumer block when response contains only constrol messages
+ ([#1115](https://github.com/IBM/sarama/pull/1115)).
+ - Add timeout config for ClusterAdmin requests
+ ([#1142](https://github.com/IBM/sarama/pull/1142)).
+ - Add version check when producing message with headers
+ ([#1117](https://github.com/IBM/sarama/pull/1117)).
+ - Fix `MetadataRequest` for empty list of topics
+ ([#1132](https://github.com/IBM/sarama/pull/1132)).
+ - Fix producer topic metadata on-demand fetch when topic error happens in metadata response
+ ([#1125](https://github.com/IBM/sarama/pull/1125)).
+
+## Version 1.17.0 (2018-05-30)
+
+New Features:
+ - Add support for gzip compression levels
+ ([#1044](https://github.com/IBM/sarama/pull/1044)).
+ - Add support for Metadata request/response pairs versions v1 to v5
+ ([#1047](https://github.com/IBM/sarama/pull/1047),
+ [#1069](https://github.com/IBM/sarama/pull/1069)).
+ - Add versioning to JoinGroup request/response pairs
+ ([#1098](https://github.com/IBM/sarama/pull/1098))
+ - Add support for CreatePartitions, DeleteGroups, DeleteRecords request/response pairs
+ ([#1065](https://github.com/IBM/sarama/pull/1065),
+ [#1096](https://github.com/IBM/sarama/pull/1096),
+ [#1027](https://github.com/IBM/sarama/pull/1027)).
+ - Add `Controller()` method to Client interface
+ ([#1063](https://github.com/IBM/sarama/pull/1063)).
+
+Improvements:
+ - ConsumerMetadataReq/Resp has been migrated to FindCoordinatorReq/Resp
+ ([#1010](https://github.com/IBM/sarama/pull/1010)).
+ - Expose missing protocol parts: `msgSet` and `recordBatch`
+ ([#1049](https://github.com/IBM/sarama/pull/1049)).
+ - Add support for v1 DeleteTopics Request
+ ([#1052](https://github.com/IBM/sarama/pull/1052)).
+ - Add support for Go 1.10
+ ([#1064](https://github.com/IBM/sarama/pull/1064)).
+ - Claim support for Kafka 1.1.0
+ ([#1073](https://github.com/IBM/sarama/pull/1073)).
+
+Bug Fixes:
+ - Fix FindCoordinatorResponse.encode to allow nil Coordinator
+ ([#1050](https://github.com/IBM/sarama/pull/1050),
+ [#1051](https://github.com/IBM/sarama/pull/1051)).
+ - Clear all metadata when we have the latest topic info
+ ([#1033](https://github.com/IBM/sarama/pull/1033)).
+ - Make `PartitionConsumer.Close` idempotent
+ ([#1092](https://github.com/IBM/sarama/pull/1092)).
+
+## Version 1.16.0 (2018-02-12)
+
+New Features:
+ - Add support for the Create/Delete Topics request/response pairs
+ ([#1007](https://github.com/IBM/sarama/pull/1007),
+ [#1008](https://github.com/IBM/sarama/pull/1008)).
+ - Add support for the Describe/Create/Delete ACL request/response pairs
+ ([#1009](https://github.com/IBM/sarama/pull/1009)).
+ - Add support for the five transaction-related request/response pairs
+ ([#1016](https://github.com/IBM/sarama/pull/1016)).
+
+Improvements:
+ - Permit setting version on mock producer responses
+ ([#999](https://github.com/IBM/sarama/pull/999)).
+ - Add `NewMockBrokerListener` helper for testing TLS connections
+ ([#1019](https://github.com/IBM/sarama/pull/1019)).
+ - Changed the default value for `Consumer.Fetch.Default` from 32KiB to 1MiB
+ which results in much higher throughput in most cases
+ ([#1024](https://github.com/IBM/sarama/pull/1024)).
+ - Reuse the `time.Ticker` across fetch requests in the PartitionConsumer to
+ reduce CPU and memory usage when processing many partitions
+ ([#1028](https://github.com/IBM/sarama/pull/1028)).
+ - Assign relative offsets to messages in the producer to save the brokers a
+ recompression pass
+ ([#1002](https://github.com/IBM/sarama/pull/1002),
+ [#1015](https://github.com/IBM/sarama/pull/1015)).
+
+Bug Fixes:
+ - Fix producing uncompressed batches with the new protocol format
+ ([#1032](https://github.com/IBM/sarama/issues/1032)).
+ - Fix consuming compacted topics with the new protocol format
+ ([#1005](https://github.com/IBM/sarama/issues/1005)).
+ - Fix consuming topics with a mix of protocol formats
+ ([#1021](https://github.com/IBM/sarama/issues/1021)).
+ - Fix consuming when the broker includes multiple batches in a single response
+ ([#1022](https://github.com/IBM/sarama/issues/1022)).
+ - Fix detection of `PartialTrailingMessage` when the partial message was
+ truncated before the magic value indicating its version
+ ([#1030](https://github.com/IBM/sarama/pull/1030)).
+ - Fix expectation-checking in the mock of `SyncProducer.SendMessages`
+ ([#1035](https://github.com/IBM/sarama/pull/1035)).
+
+## Version 1.15.0 (2017-12-08)
+
+New Features:
+ - Claim official support for Kafka 1.0, though it did already work
+ ([#984](https://github.com/IBM/sarama/pull/984)).
+ - Helper methods for Kafka version numbers to/from strings
+ ([#989](https://github.com/IBM/sarama/pull/989)).
+ - Implement CreatePartitions request/response
+ ([#985](https://github.com/IBM/sarama/pull/985)).
+
+Improvements:
+ - Add error codes 45-60
+ ([#986](https://github.com/IBM/sarama/issues/986)).
+
+Bug Fixes:
+ - Fix slow consuming for certain Kafka 0.11/1.0 configurations
+ ([#982](https://github.com/IBM/sarama/pull/982)).
+ - Correctly determine when a FetchResponse contains the new message format
+ ([#990](https://github.com/IBM/sarama/pull/990)).
+ - Fix producing with multiple headers
+ ([#996](https://github.com/IBM/sarama/pull/996)).
+ - Fix handling of truncated record batches
+ ([#998](https://github.com/IBM/sarama/pull/998)).
+ - Fix leaking metrics when closing brokers
+ ([#991](https://github.com/IBM/sarama/pull/991)).
+
+## Version 1.14.0 (2017-11-13)
+
+New Features:
+ - Add support for the new Kafka 0.11 record-batch format, including the wire
+ protocol and the necessary behavioural changes in the producer and consumer.
+ Transactions and idempotency are not yet supported, but producing and
+ consuming should work with all the existing bells and whistles (batching,
+ compression, etc) as well as the new custom headers. Thanks to Vlad Hanciuta
+ of Arista Networks for this work. Part of
+ ([#901](https://github.com/IBM/sarama/issues/901)).
+
+Bug Fixes:
+ - Fix encoding of ProduceResponse versions in test
+ ([#970](https://github.com/IBM/sarama/pull/970)).
+ - Return partial replicas list when we have it
+ ([#975](https://github.com/IBM/sarama/pull/975)).
+
+## Version 1.13.0 (2017-10-04)
+
+New Features:
+ - Support for FetchRequest version 3
+ ([#905](https://github.com/IBM/sarama/pull/905)).
+ - Permit setting version on mock FetchResponses
+ ([#939](https://github.com/IBM/sarama/pull/939)).
+ - Add a configuration option to support storing only minimal metadata for
+ extremely large clusters
+ ([#937](https://github.com/IBM/sarama/pull/937)).
+ - Add `PartitionOffsetManager.ResetOffset` for backtracking tracked offsets
+ ([#932](https://github.com/IBM/sarama/pull/932)).
+
+Improvements:
+ - Provide the block-level timestamp when consuming compressed messages
+ ([#885](https://github.com/IBM/sarama/issues/885)).
+ - `Client.Replicas` and `Client.InSyncReplicas` now respect the order returned
+ by the broker, which can be meaningful
+ ([#930](https://github.com/IBM/sarama/pull/930)).
+ - Use a `Ticker` to reduce consumer timer overhead at the cost of higher
+ variance in the actual timeout
+ ([#933](https://github.com/IBM/sarama/pull/933)).
+
+Bug Fixes:
+ - Gracefully handle messages with negative timestamps
+ ([#907](https://github.com/IBM/sarama/pull/907)).
+ - Raise a proper error when encountering an unknown message version
+ ([#940](https://github.com/IBM/sarama/pull/940)).
+
+## Version 1.12.0 (2017-05-08)
+
+New Features:
+ - Added support for the `ApiVersions` request and response pair, and Kafka
+ version 0.10.2 ([#867](https://github.com/IBM/sarama/pull/867)). Note
+ that you still need to specify the Kafka version in the Sarama configuration
+ for the time being.
+ - Added a `Brokers` method to the Client which returns the complete set of
+ active brokers ([#813](https://github.com/IBM/sarama/pull/813)).
+ - Added an `InSyncReplicas` method to the Client which returns the set of all
+ in-sync broker IDs for the given partition, now that the Kafka versions for
+ which this was misleading are no longer in our supported set
+ ([#872](https://github.com/IBM/sarama/pull/872)).
+ - Added a `NewCustomHashPartitioner` method which allows constructing a hash
+ partitioner with a custom hash method in case the default (FNV-1a) is not
+ suitable
+ ([#837](https://github.com/IBM/sarama/pull/837),
+ [#841](https://github.com/IBM/sarama/pull/841)).
+
+Improvements:
+ - Recognize more Kafka error codes
+ ([#859](https://github.com/IBM/sarama/pull/859)).
+
+Bug Fixes:
+ - Fix an issue where decoding a malformed FetchRequest would not return the
+ correct error ([#818](https://github.com/IBM/sarama/pull/818)).
+ - Respect ordering of group protocols in JoinGroupRequests. This fix is
+ transparent if you're using the `AddGroupProtocol` or
+ `AddGroupProtocolMetadata` helpers; otherwise you will need to switch from
+ the `GroupProtocols` field (now deprecated) to use `OrderedGroupProtocols`
+ ([#812](https://github.com/IBM/sarama/issues/812)).
+ - Fix an alignment-related issue with atomics on 32-bit architectures
+ ([#859](https://github.com/IBM/sarama/pull/859)).
+
+## Version 1.11.0 (2016-12-20)
+
+_Important:_ As of Sarama 1.11 it is necessary to set the config value of
+`Producer.Return.Successes` to true in order to use the SyncProducer. Previous
+versions would silently override this value when instantiating a SyncProducer
+which led to unexpected values and data races.
+
+New Features:
+ - Metrics! Thanks to Sébastien Launay for all his work on this feature
+ ([#701](https://github.com/IBM/sarama/pull/701),
+ [#746](https://github.com/IBM/sarama/pull/746),
+ [#766](https://github.com/IBM/sarama/pull/766)).
+ - Add support for LZ4 compression
+ ([#786](https://github.com/IBM/sarama/pull/786)).
+ - Add support for ListOffsetRequest v1 and Kafka 0.10.1
+ ([#775](https://github.com/IBM/sarama/pull/775)).
+ - Added a `HighWaterMarks` method to the Consumer which aggregates the
+ `HighWaterMarkOffset` values of its child topic/partitions
+ ([#769](https://github.com/IBM/sarama/pull/769)).
+
+Bug Fixes:
+ - Fixed producing when using timestamps, compression and Kafka 0.10
+ ([#759](https://github.com/IBM/sarama/pull/759)).
+ - Added missing decoder methods to DescribeGroups response
+ ([#756](https://github.com/IBM/sarama/pull/756)).
+ - Fix producer shutdown when `Return.Errors` is disabled
+ ([#787](https://github.com/IBM/sarama/pull/787)).
+ - Don't mutate configuration in SyncProducer
+ ([#790](https://github.com/IBM/sarama/pull/790)).
+ - Fix crash on SASL initialization failure
+ ([#795](https://github.com/IBM/sarama/pull/795)).
+
+## Version 1.10.1 (2016-08-30)
+
+Bug Fixes:
+ - Fix the documentation for `HashPartitioner` which was incorrect
+ ([#717](https://github.com/IBM/sarama/pull/717)).
+ - Permit client creation even when it is limited by ACLs
+ ([#722](https://github.com/IBM/sarama/pull/722)).
+ - Several fixes to the consumer timer optimization code, regressions introduced
+ in v1.10.0. Go's timers are finicky
+ ([#730](https://github.com/IBM/sarama/pull/730),
+ [#733](https://github.com/IBM/sarama/pull/733),
+ [#734](https://github.com/IBM/sarama/pull/734)).
+ - Handle consuming compressed relative offsets with Kafka 0.10
+ ([#735](https://github.com/IBM/sarama/pull/735)).
+
+## Version 1.10.0 (2016-08-02)
+
+_Important:_ As of Sarama 1.10 it is necessary to tell Sarama the version of
+Kafka you are running against (via the `config.Version` value) in order to use
+features that may not be compatible with old Kafka versions. If you don't
+specify this value it will default to 0.8.2 (the minimum supported), and trying
+to use more recent features (like the offset manager) will fail with an error.
+
+_Also:_ The offset-manager's behaviour has been changed to match the upstream
+java consumer (see [#705](https://github.com/IBM/sarama/pull/705) and
+[#713](https://github.com/IBM/sarama/pull/713)). If you use the
+offset-manager, please ensure that you are committing one *greater* than the
+last consumed message offset or else you may end up consuming duplicate
+messages.
+
+New Features:
+ - Support for Kafka 0.10
+ ([#672](https://github.com/IBM/sarama/pull/672),
+ [#678](https://github.com/IBM/sarama/pull/678),
+ [#681](https://github.com/IBM/sarama/pull/681), and others).
+ - Support for configuring the target Kafka version
+ ([#676](https://github.com/IBM/sarama/pull/676)).
+ - Batch producing support in the SyncProducer
+ ([#677](https://github.com/IBM/sarama/pull/677)).
+ - Extend producer mock to allow setting expectations on message contents
+ ([#667](https://github.com/IBM/sarama/pull/667)).
+
+Improvements:
+ - Support `nil` compressed messages for deleting in compacted topics
+ ([#634](https://github.com/IBM/sarama/pull/634)).
+ - Pre-allocate decoding errors, greatly reducing heap usage and GC time against
+ misbehaving brokers ([#690](https://github.com/IBM/sarama/pull/690)).
+ - Re-use consumer expiry timers, removing one allocation per consumed message
+ ([#707](https://github.com/IBM/sarama/pull/707)).
+
+Bug Fixes:
+ - Actually default the client ID to "sarama" like we say we do
+ ([#664](https://github.com/IBM/sarama/pull/664)).
+ - Fix a rare issue where `Client.Leader` could return the wrong error
+ ([#685](https://github.com/IBM/sarama/pull/685)).
+ - Fix a possible tight loop in the consumer
+ ([#693](https://github.com/IBM/sarama/pull/693)).
+ - Match upstream's offset-tracking behaviour
+ ([#705](https://github.com/IBM/sarama/pull/705)).
+ - Report UnknownTopicOrPartition errors from the offset manager
+ ([#706](https://github.com/IBM/sarama/pull/706)).
+ - Fix possible negative partition value from the HashPartitioner
+ ([#709](https://github.com/IBM/sarama/pull/709)).
+
+## Version 1.9.0 (2016-05-16)
+
+New Features:
+ - Add support for custom offset manager retention durations
+ ([#602](https://github.com/IBM/sarama/pull/602)).
+ - Publish low-level mocks to enable testing of third-party producer/consumer
+ implementations ([#570](https://github.com/IBM/sarama/pull/570)).
+ - Declare support for Golang 1.6
+ ([#611](https://github.com/IBM/sarama/pull/611)).
+ - Support for SASL plain-text auth
+ ([#648](https://github.com/IBM/sarama/pull/648)).
+
+Improvements:
+ - Simplified broker locking scheme slightly
+ ([#604](https://github.com/IBM/sarama/pull/604)).
+ - Documentation cleanup
+ ([#605](https://github.com/IBM/sarama/pull/605),
+ [#621](https://github.com/IBM/sarama/pull/621),
+ [#654](https://github.com/IBM/sarama/pull/654)).
+
+Bug Fixes:
+ - Fix race condition shutting down the OffsetManager
+ ([#658](https://github.com/IBM/sarama/pull/658)).
+
+## Version 1.8.0 (2016-02-01)
+
+New Features:
+ - Full support for Kafka 0.9:
+ - All protocol messages and fields
+ ([#586](https://github.com/IBM/sarama/pull/586),
+ [#588](https://github.com/IBM/sarama/pull/588),
+ [#590](https://github.com/IBM/sarama/pull/590)).
+ - Verified that TLS support works
+ ([#581](https://github.com/IBM/sarama/pull/581)).
+ - Fixed the OffsetManager compatibility
+ ([#585](https://github.com/IBM/sarama/pull/585)).
+
+Improvements:
+ - Optimize for fewer system calls when reading from the network
+ ([#584](https://github.com/IBM/sarama/pull/584)).
+ - Automatically retry `InvalidMessage` errors to match upstream behaviour
+ ([#589](https://github.com/IBM/sarama/pull/589)).
+
+## Version 1.7.0 (2015-12-11)
+
+New Features:
+ - Preliminary support for Kafka 0.9
+ ([#572](https://github.com/IBM/sarama/pull/572)). This comes with several
+ caveats:
+ - Protocol-layer support is mostly in place
+ ([#577](https://github.com/IBM/sarama/pull/577)), however Kafka 0.9
+ renamed some messages and fields, which we did not in order to preserve API
+ compatibility.
+ - The producer and consumer work against 0.9, but the offset manager does
+ not ([#573](https://github.com/IBM/sarama/pull/573)).
+ - TLS support may or may not work
+ ([#581](https://github.com/IBM/sarama/pull/581)).
+
+Improvements:
+ - Don't wait for request timeouts on dead brokers, greatly speeding recovery
+ when the TCP connection is left hanging
+ ([#548](https://github.com/IBM/sarama/pull/548)).
+ - Refactored part of the producer. The new version provides a much more elegant
+ solution to [#449](https://github.com/IBM/sarama/pull/449). It is also
+ slightly more efficient, and much more precise in calculating batch sizes
+ when compression is used
+ ([#549](https://github.com/IBM/sarama/pull/549),
+ [#550](https://github.com/IBM/sarama/pull/550),
+ [#551](https://github.com/IBM/sarama/pull/551)).
+
+Bug Fixes:
+ - Fix race condition in consumer test mock
+ ([#553](https://github.com/IBM/sarama/pull/553)).
+
+## Version 1.6.1 (2015-09-25)
+
+Bug Fixes:
+ - Fix panic that could occur if a user-supplied message value failed to encode
+ ([#449](https://github.com/IBM/sarama/pull/449)).
+
+## Version 1.6.0 (2015-09-04)
+
+New Features:
+ - Implementation of a consumer offset manager using the APIs introduced in
+ Kafka 0.8.2. The API is designed mainly for integration into a future
+ high-level consumer, not for direct use, although it is *possible* to use it
+ directly.
+ ([#461](https://github.com/IBM/sarama/pull/461)).
+
+Improvements:
+ - CRC32 calculation is much faster on machines with SSE4.2 instructions,
+ removing a major hotspot from most profiles
+ ([#255](https://github.com/IBM/sarama/pull/255)).
+
+Bug Fixes:
+ - Make protocol decoding more robust against some malformed packets generated
+ by go-fuzz ([#523](https://github.com/IBM/sarama/pull/523),
+ [#525](https://github.com/IBM/sarama/pull/525)) or found in other ways
+ ([#528](https://github.com/IBM/sarama/pull/528)).
+ - Fix a potential race condition panic in the consumer on shutdown
+ ([#529](https://github.com/IBM/sarama/pull/529)).
+
+## Version 1.5.0 (2015-08-17)
+
+New Features:
+ - TLS-encrypted network connections are now supported. This feature is subject
+ to change when Kafka releases built-in TLS support, but for now this is
+ enough to work with TLS-terminating proxies
+ ([#154](https://github.com/IBM/sarama/pull/154)).
+
+Improvements:
+ - The consumer will not block if a single partition is not drained by the user;
+ all other partitions will continue to consume normally
+ ([#485](https://github.com/IBM/sarama/pull/485)).
+ - Formatting of error strings has been much improved
+ ([#495](https://github.com/IBM/sarama/pull/495)).
+ - Internal refactoring of the producer for code cleanliness and to enable
+ future work ([#300](https://github.com/IBM/sarama/pull/300)).
+
+Bug Fixes:
+ - Fix a potential deadlock in the consumer on shutdown
+ ([#475](https://github.com/IBM/sarama/pull/475)).
+
+## Version 1.4.3 (2015-07-21)
+
+Bug Fixes:
+ - Don't include the partitioner in the producer's "fetch partitions"
+ circuit-breaker ([#466](https://github.com/IBM/sarama/pull/466)).
+ - Don't retry messages until the broker is closed when abandoning a broker in
+ the producer ([#468](https://github.com/IBM/sarama/pull/468)).
+ - Update the import path for snappy-go, it has moved again and the API has
+ changed slightly ([#486](https://github.com/IBM/sarama/pull/486)).
+
+## Version 1.4.2 (2015-05-27)
+
+Bug Fixes:
+ - Update the import path for snappy-go, it has moved from google code to github
+ ([#456](https://github.com/IBM/sarama/pull/456)).
+
+## Version 1.4.1 (2015-05-25)
+
+Improvements:
+ - Optimizations when decoding snappy messages, thanks to John Potocny
+ ([#446](https://github.com/IBM/sarama/pull/446)).
+
+Bug Fixes:
+ - Fix hypothetical race conditions on producer shutdown
+ ([#450](https://github.com/IBM/sarama/pull/450),
+ [#451](https://github.com/IBM/sarama/pull/451)).
+
+## Version 1.4.0 (2015-05-01)
+
+New Features:
+ - The consumer now implements `Topics()` and `Partitions()` methods to enable
+ users to dynamically choose what topics/partitions to consume without
+ instantiating a full client
+ ([#431](https://github.com/IBM/sarama/pull/431)).
+ - The partition-consumer now exposes the high water mark offset value returned
+ by the broker via the `HighWaterMarkOffset()` method ([#339](https://github.com/IBM/sarama/pull/339)).
+ - Added a `kafka-console-consumer` tool capable of handling multiple
+ partitions, and deprecated the now-obsolete `kafka-console-partitionConsumer`
+ ([#439](https://github.com/IBM/sarama/pull/439),
+ [#442](https://github.com/IBM/sarama/pull/442)).
+
+Improvements:
+ - The producer's logging during retry scenarios is more consistent, more
+ useful, and slightly less verbose
+ ([#429](https://github.com/IBM/sarama/pull/429)).
+ - The client now shuffles its initial list of seed brokers in order to prevent
+ thundering herd on the first broker in the list
+ ([#441](https://github.com/IBM/sarama/pull/441)).
+
+Bug Fixes:
+ - The producer now correctly manages its state if retries occur when it is
+ shutting down, fixing several instances of confusing behaviour and at least
+ one potential deadlock ([#419](https://github.com/IBM/sarama/pull/419)).
+ - The consumer now handles messages for different partitions asynchronously,
+ making it much more resilient to specific user code ordering
+ ([#325](https://github.com/IBM/sarama/pull/325)).
+
+## Version 1.3.0 (2015-04-16)
+
+New Features:
+ - The client now tracks consumer group coordinators using
+ ConsumerMetadataRequests similar to how it tracks partition leadership using
+ regular MetadataRequests ([#411](https://github.com/IBM/sarama/pull/411)).
+ This adds two methods to the client API:
+ - `Coordinator(consumerGroup string) (*Broker, error)`
+ - `RefreshCoordinator(consumerGroup string) error`
+
+Improvements:
+ - ConsumerMetadataResponses now automatically create a Broker object out of the
+ ID/address/port combination for the Coordinator; accessing the fields
+ individually has been deprecated
+ ([#413](https://github.com/IBM/sarama/pull/413)).
+ - Much improved handling of `OffsetOutOfRange` errors in the consumer.
+ Consumers will fail to start if the provided offset is out of range
+ ([#418](https://github.com/IBM/sarama/pull/418))
+ and they will automatically shut down if the offset falls out of range
+ ([#424](https://github.com/IBM/sarama/pull/424)).
+ - Small performance improvement in encoding and decoding protocol messages
+ ([#427](https://github.com/IBM/sarama/pull/427)).
+
+Bug Fixes:
+ - Fix a rare race condition in the client's background metadata refresher if
+ it happens to be activated while the client is being closed
+ ([#422](https://github.com/IBM/sarama/pull/422)).
+
+## Version 1.2.0 (2015-04-07)
+
+Improvements:
+ - The producer's behaviour when `Flush.Frequency` is set is now more intuitive
+ ([#389](https://github.com/IBM/sarama/pull/389)).
+ - The producer is now somewhat more memory-efficient during and after retrying
+ messages due to an improved queue implementation
+ ([#396](https://github.com/IBM/sarama/pull/396)).
+ - The consumer produces much more useful logging output when leadership
+ changes ([#385](https://github.com/IBM/sarama/pull/385)).
+ - The client's `GetOffset` method will now automatically refresh metadata and
+ retry once in the event of stale information or similar
+ ([#394](https://github.com/IBM/sarama/pull/394)).
+ - Broker connections now have support for using TCP keepalives
+ ([#407](https://github.com/IBM/sarama/issues/407)).
+
+Bug Fixes:
+ - The OffsetCommitRequest message now correctly implements all three possible
+ API versions ([#390](https://github.com/IBM/sarama/pull/390),
+ [#400](https://github.com/IBM/sarama/pull/400)).
+
+## Version 1.1.0 (2015-03-20)
+
+Improvements:
+ - Wrap the producer's partitioner call in a circuit-breaker so that repeatedly
+ broken topics don't choke throughput
+ ([#373](https://github.com/IBM/sarama/pull/373)).
+
+Bug Fixes:
+ - Fix the producer's internal reference counting in certain unusual scenarios
+ ([#367](https://github.com/IBM/sarama/pull/367)).
+ - Fix the consumer's internal reference counting in certain unusual scenarios
+ ([#369](https://github.com/IBM/sarama/pull/369)).
+ - Fix a condition where the producer's internal control messages could have
+ gotten stuck ([#368](https://github.com/IBM/sarama/pull/368)).
+ - Fix an issue where invalid partition lists would be cached when asking for
+ metadata for a non-existant topic ([#372](https://github.com/IBM/sarama/pull/372)).
+
+
+## Version 1.0.0 (2015-03-17)
+
+Version 1.0.0 is the first tagged version, and is almost a complete rewrite. The primary differences with previous untagged versions are:
+
+- The producer has been rewritten; there is now a `SyncProducer` with a blocking API, and an `AsyncProducer` that is non-blocking.
+- The consumer has been rewritten to only open one connection per broker instead of one connection per partition.
+- The main types of Sarama are now interfaces to make depedency injection easy; mock implementations for `Consumer`, `SyncProducer` and `AsyncProducer` are provided in the `github.com/IBM/sarama/mocks` package.
+- For most uses cases, it is no longer necessary to open a `Client`; this will be done for you.
+- All the configuration values have been unified in the `Config` struct.
+- Much improved test suite.
diff --git a/vendor/github.com/IBM/sarama/CODE_OF_CONDUCT.md b/vendor/github.com/IBM/sarama/CODE_OF_CONDUCT.md
new file mode 100644
index 0000000..8470ec5
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/CODE_OF_CONDUCT.md
@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+ and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+ overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+ advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+ address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+dominic.evans@uk.ibm.com.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.
diff --git a/vendor/github.com/IBM/sarama/CONTRIBUTING.md b/vendor/github.com/IBM/sarama/CONTRIBUTING.md
new file mode 100644
index 0000000..bb88127
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/CONTRIBUTING.md
@@ -0,0 +1,77 @@
+# Contributing
+
+[fork]: https://github.com/IBM/sarama/fork
+[pr]: https://github.com/IBM/sarama/compare
+[released]: https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license
+
+Hi there! We are thrilled that you would like to contribute to Sarama.
+Contributions are always welcome, both reporting issues and submitting pull requests!
+
+## Reporting issues
+
+Please make sure to include any potentially useful information in the issue, so we can pinpoint the issue faster without going back and forth.
+
+- What SHA of Sarama are you running? If this is not the latest SHA on the main branch, please try if the problem persists with the latest version.
+- You can set `sarama.Logger` to a [log.Logger](http://golang.org/pkg/log/#Logger) instance to capture debug output. Please include it in your issue description.
+- Also look at the logs of the Kafka broker you are connected to. If you see anything out of the ordinary, please include it.
+
+Also, please include the following information about your environment, so we can help you faster:
+
+- What version of Kafka are you using?
+- What version of Go are you using?
+- What are the values of your Producer/Consumer/Client configuration?
+
+
+## Contributing a change
+
+Contributions to this project are [released][released] to the public under the project's [opensource license](LICENSE.md).
+By contributing to this project you agree to the [Developer Certificate of Origin](https://developercertificate.org/) (DCO).
+The DCO was created by the Linux Kernel community and is a simple statement that you, as a contributor, wrote or otherwise have the legal right to contribute those changes.
+
+Contributors must _sign-off_ that they adhere to these requirements by adding a `Signed-off-by` line to all commit messages with an email address that matches the commit author:
+
+```
+feat: this is my commit message
+
+Signed-off-by: Random J Developer <random@developer.example.org>
+```
+
+Git even has a `-s` command line option to append this automatically to your
+commit message:
+
+```
+$ git commit -s -m 'This is my commit message'
+```
+
+Because this library is in production use by many people and applications, we code review all additions.
+To make the review process go as smooth as possible, please consider the following.
+
+- If you plan to work on something major, please open an issue to discuss the design first.
+- Don't break backwards compatibility. If you really have to, open an issue to discuss this first.
+- Make sure to use the `go fmt` command to format your code according to the standards. Even better, set up your editor to do this for you when saving.
+- Run [go vet](https://golang.org/cmd/vet/) to detect any suspicious constructs in your code that could be bugs.
+- Explicitly handle all error return values. If you really want to ignore an error value, you can assign it to `_`. You can use [errcheck](https://github.com/kisielk/errcheck) to verify whether you have handled all errors.
+- You may also want to run [golint](https://github.com/golang/lint) as well to detect style problems.
+- Add tests that cover the changes you made. Make sure to run `go test` with the `-race` argument to test for race conditions.
+- Make sure your code is supported by all the Go versions we support.
+ You can rely on GitHub Actions for testing older Go versions.
+
+## Submitting a pull request
+
+0. [Fork][fork] and clone the repository
+1. Create a new branch: `git checkout -b my-branch-name`
+2. Make your change, push to your fork and [submit a pull request][pr]
+3. Wait for your pull request to be reviewed and merged.
+
+Here are a few things you can do that will increase the likelihood of your pull request being accepted:
+
+- Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
+- Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+
+## Further Reading
+
+- [Developer Certificate of Origin versus Contributor License Agreements](https://julien.ponge.org/blog/developer-certificate-of-origin-versus-contributor-license-agreements/)
+- [The most powerful contributor agreement](https://lwn.net/Articles/592503/)
+- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)
+- [Using Pull Requests](https://help.github.com/articles/about-pull-requests/)
+- [GitHub Help](https://help.github.com)
diff --git a/vendor/github.com/IBM/sarama/Dockerfile.kafka b/vendor/github.com/IBM/sarama/Dockerfile.kafka
new file mode 100644
index 0000000..bb18b6c
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/Dockerfile.kafka
@@ -0,0 +1,55 @@
+FROM registry.access.redhat.com/ubi9/ubi-minimal:9.6@sha256:7c5495d5fad59aaee12abc3cbbd2b283818ee1e814b00dbc7f25bf2d14fa4f0c
+
+USER root
+
+RUN microdnf update -y \
+ && microdnf install -y git gzip java-17-openjdk-headless tar tzdata-java \
+ && microdnf reinstall -y tzdata \
+ && microdnf clean all
+
+ENV JAVA_HOME=/usr/lib/jvm/jre-17
+
+# https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html
+# Ensure Java doesn't cache any dns results
+RUN cd /etc/java/java-17-openjdk/*/conf/security \
+ && sed -e '/networkaddress.cache.ttl/d' -e '/networkaddress.cache.negative.ttl/d' -i java.security \
+ && echo 'networkaddress.cache.ttl=0' >> java.security \
+ && echo 'networkaddress.cache.negative.ttl=0' >> java.security
+
+ARG SCALA_VERSION="2.13"
+ARG KAFKA_VERSION="3.9.1"
+
+WORKDIR /tmp
+
+# https://github.com/apache/kafka/blob/2e2b0a58eda3e677763af974a44a6aaa3c280214/tests/docker/Dockerfile#L77-L105
+ARG KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages"
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN --mount=type=bind,target=.,rw=true \
+ mkdir -p "/opt/kafka-${KAFKA_VERSION}" \
+ && chmod a+rw "/opt/kafka-${KAFKA_VERSION}" \
+ && curl -s "$KAFKA_MIRROR/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" | tar xz --strip-components=1 -C "/opt/kafka-${KAFKA_VERSION}"
+
+# older kafka versions depend upon jaxb-api being bundled with the JDK, but it
+# was removed from Java 11 so work around that by including it in the kafka
+# libs dir regardless
+RUN curl -sLO "https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0.jar" \
+ && for DIR in /opt/kafka-*; do cp -v jaxb-api-2.3.0.jar $DIR/libs/ ; done \
+ && rm -f jaxb-api-2.3.0.jar
+
+# older kafka versions with the zookeeper 3.4.13/3.4.14 client aren't compatible with Java 17 so quietly bump them to 3.5.9
+RUN if ! stat /opt/kafka-${KAFKA_VERSION}/libs/zookeeper-3.4.*.jar; then exit 0; fi ; \
+ rm -f /opt/kafka-${KAFKA_VERSION}/libs/zookeeper-3.4.*.jar \
+ && curl --fail -sSL -o "/opt/kafka-${KAFKA_VERSION}/libs/zookeeper-3.5.9.jar" "https://repo1.maven.org/maven2/org/apache/zookeeper/zookeeper/3.5.9/zookeeper-3.5.9.jar" \
+ && curl --fail -sSL -o "/opt/kafka-${KAFKA_VERSION}/libs/zookeeper-jute-3.5.9.jar" "https://repo1.maven.org/maven2/org/apache/zookeeper/zookeeper-jute/3.5.9/zookeeper-jute-3.5.9.jar"
+
+WORKDIR /opt/kafka-${KAFKA_VERSION}
+
+ENV JAVA_MAJOR_VERSION=17
+
+RUN sed -e "s/JAVA_MAJOR_VERSION=.*/JAVA_MAJOR_VERSION=${JAVA_MAJOR_VERSION}/" -i"" ./bin/kafka-run-class.sh
+
+COPY entrypoint.sh /
+
+USER 65534:65534
+
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/vendor/github.com/IBM/sarama/LICENSE.md b/vendor/github.com/IBM/sarama/LICENSE.md
new file mode 100644
index 0000000..f8f64d4
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/LICENSE.md
@@ -0,0 +1,24 @@
+# MIT License
+
+Copyright (c) 2013 Shopify
+
+Copyright (c) 2023 IBM Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/vendor/github.com/IBM/sarama/Makefile b/vendor/github.com/IBM/sarama/Makefile
new file mode 100644
index 0000000..c41445c
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/Makefile
@@ -0,0 +1,47 @@
+default: fmt get update test lint
+
+GO := go
+GOBIN := $(shell pwd)/bin
+GOBUILD := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG)
+
+GOTESTSUM := $(GOBIN)/gotestsum
+GOTESTSUM_VERSION := v1.13.0
+$(GOTESTSUM):
+ GOBIN=$(GOBIN) go install gotest.tools/gotestsum@$(GOTESTSUM_VERSION)
+
+TESTSTAT := $(GOBIN)/teststat
+TESTSTAT_VERSION := v0.1.26
+$(TESTSTAT):
+ GOBIN=$(GOBIN) go install github.com/vearutop/teststat@$(TESTSTAT_VERSION)
+
+FILES := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -not -name '*_test.go')
+TESTS := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -name '*_test.go')
+
+get:
+ $(GO) get ./...
+ $(GO) mod verify
+ $(GO) mod tidy
+
+update:
+ $(GO) get -u -v ./...
+ $(GO) mod verify
+ $(GO) mod tidy
+
+fmt:
+ gofmt -s -l -w $(FILES) $(TESTS)
+
+lint:
+ GOFLAGS="-tags=functional" golangci-lint run
+
+test: $(GOTESTSUM) $(TESTSTAT) $(TPARSE)
+ @$(GOTESTSUM) $(if ${CI},--format github-actions,--format testdox) --jsonfile _test/unittests.json --junitfile _test/unittests.xml \
+ --rerun-fails --packages="./..." \
+ -- -v -race -coverprofile=profile.out -covermode=atomic -timeout 2m
+ @$(TESTSTAT) _test/unittests.json
+
+.PHONY: test_functional
+test_functional: $(GOTESTSUM) $(TESTSTAT) $(TPARSE)
+ @$(GOTESTSUM) $(if ${CI},--format github-actions,--format testdox) --jsonfile _test/fvt.json --junitfile _test/fvt.xml \
+ --rerun-fails --packages="./..." \
+ -- -v -race -coverprofile=profile.out -covermode=atomic -timeout 15m -tags=functional
+ @$(TESTSTAT) _test/fvt.json
diff --git a/vendor/github.com/IBM/sarama/README.md b/vendor/github.com/IBM/sarama/README.md
new file mode 100644
index 0000000..4534d7b
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/README.md
@@ -0,0 +1,35 @@
+# sarama
+
+[](https://pkg.go.dev/github.com/IBM/sarama)
+[](https://securityscorecards.dev/viewer/?uri=github.com/IBM/sarama)
+[](https://www.bestpractices.dev/projects/7996)
+
+Sarama is an MIT-licensed Go client library for [Apache Kafka](https://kafka.apache.org/).
+
+## Getting started
+
+- API documentation and examples are available via [pkg.go.dev](https://pkg.go.dev/github.com/IBM/sarama).
+- Mocks for testing are available in the [mocks](./mocks) subpackage.
+- The [examples](./examples) directory contains more elaborate example applications.
+- The [tools](./tools) directory contains command line tools that can be useful for testing, diagnostics, and instrumentation.
+
+You might also want to look at the [Frequently Asked Questions](https://github.com/IBM/sarama/wiki/Frequently-Asked-Questions).
+
+## Compatibility and API stability
+
+Sarama provides a "2 releases + 2 months" compatibility guarantee: we support
+the two latest stable releases of Kafka and Go, and we provide a two month
+grace period for older releases. However, older releases of Kafka are still likely to work.
+
+Sarama follows semantic versioning and provides API stability via the standard Go
+[module version numbering](https://go.dev/doc/modules/version-numbers) scheme.
+
+A changelog is available [here](CHANGELOG.md).
+
+## Contributing
+
+- Get started by checking our [contribution guidelines](https://github.com/IBM/sarama/blob/main/CONTRIBUTING.md).
+- Read the [Sarama wiki](https://github.com/IBM/sarama/wiki) for more technical and design details.
+- The [Kafka Protocol Specification](https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol) contains a wealth of useful information.
+- For more general issues, there is [a google group](https://groups.google.com/forum/#!forum/kafka-clients) for Kafka client developers.
+- If you have any questions, just ask!
diff --git a/vendor/github.com/IBM/sarama/SECURITY.md b/vendor/github.com/IBM/sarama/SECURITY.md
new file mode 100644
index 0000000..b2f6e61
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/SECURITY.md
@@ -0,0 +1,11 @@
+# Security
+
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+The easiest way to report a security issue is privately through GitHub [here](https://github.com/IBM/sarama/security/advisories/new).
+
+See [Privately reporting a security vulnerability](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability) for full instructions.
+
+Alternatively, you can report them via e-mail or anonymous form to the IBM Product Security Incident Response Team (PSIRT) following the guidelines under the [IBM Security Vulnerability Management](https://www.ibm.com/support/pages/ibm-security-vulnerability-management) pages.
diff --git a/vendor/github.com/IBM/sarama/Vagrantfile b/vendor/github.com/IBM/sarama/Vagrantfile
new file mode 100644
index 0000000..07d7ffb
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/Vagrantfile
@@ -0,0 +1,14 @@
+# We have 5 * 192MB ZK processes and 5 * 320MB Kafka processes => 2560MB
+MEMORY = 3072
+
+Vagrant.configure("2") do |config|
+ config.vm.box = "ubuntu/bionic64"
+
+ config.vm.provision :shell, path: "vagrant/provision.sh"
+
+ config.vm.network "private_network", ip: "192.168.100.67"
+
+ config.vm.provider "virtualbox" do |v|
+ v.memory = MEMORY
+ end
+end
diff --git a/vendor/github.com/IBM/sarama/acl_bindings.go b/vendor/github.com/IBM/sarama/acl_bindings.go
new file mode 100644
index 0000000..13440be
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/acl_bindings.go
@@ -0,0 +1,138 @@
+package sarama
+
+// Resource holds information about acl resource type
+type Resource struct {
+ ResourceType AclResourceType
+ ResourceName string
+ ResourcePatternType AclResourcePatternType
+}
+
+func (r *Resource) encode(pe packetEncoder, version int16) error {
+ pe.putInt8(int8(r.ResourceType))
+
+ if err := pe.putString(r.ResourceName); err != nil {
+ return err
+ }
+
+ if version == 1 {
+ if r.ResourcePatternType == AclPatternUnknown {
+ Logger.Print("Cannot encode an unknown resource pattern type, using Literal instead")
+ r.ResourcePatternType = AclPatternLiteral
+ }
+ pe.putInt8(int8(r.ResourcePatternType))
+ }
+
+ return nil
+}
+
+func (r *Resource) decode(pd packetDecoder, version int16) (err error) {
+ resourceType, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ r.ResourceType = AclResourceType(resourceType)
+
+ if r.ResourceName, err = pd.getString(); err != nil {
+ return err
+ }
+ if version == 1 {
+ pattern, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ r.ResourcePatternType = AclResourcePatternType(pattern)
+ }
+
+ return nil
+}
+
+// Acl holds information about acl type
+type Acl struct {
+ Principal string
+ Host string
+ Operation AclOperation
+ PermissionType AclPermissionType
+}
+
+func (a *Acl) encode(pe packetEncoder) error {
+ if err := pe.putString(a.Principal); err != nil {
+ return err
+ }
+
+ if err := pe.putString(a.Host); err != nil {
+ return err
+ }
+
+ pe.putInt8(int8(a.Operation))
+ pe.putInt8(int8(a.PermissionType))
+
+ return nil
+}
+
+func (a *Acl) decode(pd packetDecoder, version int16) (err error) {
+ if a.Principal, err = pd.getString(); err != nil {
+ return err
+ }
+
+ if a.Host, err = pd.getString(); err != nil {
+ return err
+ }
+
+ operation, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ a.Operation = AclOperation(operation)
+
+ permissionType, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ a.PermissionType = AclPermissionType(permissionType)
+
+ return nil
+}
+
+// ResourceAcls is an acl resource type
+type ResourceAcls struct {
+ Resource
+ Acls []*Acl
+}
+
+func (r *ResourceAcls) encode(pe packetEncoder, version int16) error {
+ if err := r.Resource.encode(pe, version); err != nil {
+ return err
+ }
+
+ if err := pe.putArrayLength(len(r.Acls)); err != nil {
+ return err
+ }
+ for _, acl := range r.Acls {
+ if err := acl.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (r *ResourceAcls) decode(pd packetDecoder, version int16) error {
+ if err := r.Resource.decode(pd, version); err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Acls = make([]*Acl, n)
+ for i := 0; i < n; i++ {
+ r.Acls[i] = new(Acl)
+ if err := r.Acls[i].decode(pd, version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/acl_create_request.go b/vendor/github.com/IBM/sarama/acl_create_request.go
new file mode 100644
index 0000000..187cee0
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/acl_create_request.go
@@ -0,0 +1,97 @@
+package sarama
+
+// CreateAclsRequest is an acl creation request
+type CreateAclsRequest struct {
+ Version int16
+ AclCreations []*AclCreation
+}
+
+func (c *CreateAclsRequest) setVersion(v int16) {
+ c.Version = v
+}
+
+func (c *CreateAclsRequest) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(c.AclCreations)); err != nil {
+ return err
+ }
+
+ for _, aclCreation := range c.AclCreations {
+ if err := aclCreation.encode(pe, c.Version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (c *CreateAclsRequest) decode(pd packetDecoder, version int16) (err error) {
+ c.Version = version
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ c.AclCreations = make([]*AclCreation, n)
+
+ for i := 0; i < n; i++ {
+ c.AclCreations[i] = new(AclCreation)
+ if err := c.AclCreations[i].decode(pd, version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (c *CreateAclsRequest) key() int16 {
+ return apiKeyCreateAcls
+}
+
+func (c *CreateAclsRequest) version() int16 {
+ return c.Version
+}
+
+func (c *CreateAclsRequest) headerVersion() int16 {
+ return 1
+}
+
+func (c *CreateAclsRequest) isValidVersion() bool {
+ return c.Version >= 0 && c.Version <= 1
+}
+
+func (c *CreateAclsRequest) requiredVersion() KafkaVersion {
+ switch c.Version {
+ case 1:
+ return V2_0_0_0
+ default:
+ return V0_11_0_0
+ }
+}
+
+// AclCreation is a wrapper around Resource and Acl type
+type AclCreation struct {
+ Resource
+ Acl
+}
+
+func (a *AclCreation) encode(pe packetEncoder, version int16) error {
+ if err := a.Resource.encode(pe, version); err != nil {
+ return err
+ }
+ if err := a.Acl.encode(pe); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (a *AclCreation) decode(pd packetDecoder, version int16) (err error) {
+ if err := a.Resource.decode(pd, version); err != nil {
+ return err
+ }
+ if err := a.Acl.decode(pd, version); err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/acl_create_response.go b/vendor/github.com/IBM/sarama/acl_create_response.go
new file mode 100644
index 0000000..514fc64
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/acl_create_response.go
@@ -0,0 +1,110 @@
+package sarama
+
+import "time"
+
+// CreateAclsResponse is a an acl response creation type
+type CreateAclsResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ AclCreationResponses []*AclCreationResponse
+}
+
+func (c *CreateAclsResponse) setVersion(v int16) {
+ c.Version = v
+}
+
+func (c *CreateAclsResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(c.ThrottleTime)
+
+ if err := pe.putArrayLength(len(c.AclCreationResponses)); err != nil {
+ return err
+ }
+
+ for _, aclCreationResponse := range c.AclCreationResponses {
+ if err := aclCreationResponse.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (c *CreateAclsResponse) decode(pd packetDecoder, version int16) (err error) {
+ c.ThrottleTime, err = pd.getDurationMs()
+ if err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ c.AclCreationResponses = make([]*AclCreationResponse, n)
+ for i := 0; i < n; i++ {
+ c.AclCreationResponses[i] = new(AclCreationResponse)
+ if err := c.AclCreationResponses[i].decode(pd, version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (c *CreateAclsResponse) key() int16 {
+ return apiKeyCreateAcls
+}
+
+func (c *CreateAclsResponse) version() int16 {
+ return c.Version
+}
+
+func (c *CreateAclsResponse) headerVersion() int16 {
+ return 0
+}
+
+func (c *CreateAclsResponse) isValidVersion() bool {
+ return c.Version >= 0 && c.Version <= 1
+}
+
+func (c *CreateAclsResponse) requiredVersion() KafkaVersion {
+ switch c.Version {
+ case 1:
+ return V2_0_0_0
+ default:
+ return V0_11_0_0
+ }
+}
+
+func (r *CreateAclsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
+
+// AclCreationResponse is an acl creation response type
+type AclCreationResponse struct {
+ Err KError
+ ErrMsg *string
+}
+
+func (a *AclCreationResponse) encode(pe packetEncoder) error {
+ pe.putKError(a.Err)
+
+ if err := pe.putNullableString(a.ErrMsg); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (a *AclCreationResponse) decode(pd packetDecoder, version int16) (err error) {
+ a.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if a.ErrMsg, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/acl_delete_request.go b/vendor/github.com/IBM/sarama/acl_delete_request.go
new file mode 100644
index 0000000..ff40740
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/acl_delete_request.go
@@ -0,0 +1,70 @@
+package sarama
+
+// DeleteAclsRequest is a delete acl request
+type DeleteAclsRequest struct {
+ Version int
+ Filters []*AclFilter
+}
+
+func (d *DeleteAclsRequest) setVersion(v int16) {
+ d.Version = int(v)
+}
+
+func (d *DeleteAclsRequest) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(d.Filters)); err != nil {
+ return err
+ }
+
+ for _, filter := range d.Filters {
+ filter.Version = d.Version
+ if err := filter.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (d *DeleteAclsRequest) decode(pd packetDecoder, version int16) (err error) {
+ d.Version = int(version)
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ d.Filters = make([]*AclFilter, n)
+ for i := 0; i < n; i++ {
+ d.Filters[i] = new(AclFilter)
+ d.Filters[i].Version = int(version)
+ if err := d.Filters[i].decode(pd, version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (d *DeleteAclsRequest) key() int16 {
+ return apiKeyDeleteAcls
+}
+
+func (d *DeleteAclsRequest) version() int16 {
+ return int16(d.Version)
+}
+
+func (d *DeleteAclsRequest) headerVersion() int16 {
+ return 1
+}
+
+func (d *DeleteAclsRequest) isValidVersion() bool {
+ return d.Version >= 0 && d.Version <= 1
+}
+
+func (d *DeleteAclsRequest) requiredVersion() KafkaVersion {
+ switch d.Version {
+ case 1:
+ return V2_0_0_0
+ default:
+ return V0_11_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/acl_delete_response.go b/vendor/github.com/IBM/sarama/acl_delete_response.go
new file mode 100644
index 0000000..bc6637e
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/acl_delete_response.go
@@ -0,0 +1,176 @@
+package sarama
+
+import "time"
+
+// DeleteAclsResponse is a delete acl response
+type DeleteAclsResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ FilterResponses []*FilterResponse
+}
+
+func (d *DeleteAclsResponse) setVersion(v int16) {
+ d.Version = v
+}
+
+func (d *DeleteAclsResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(d.ThrottleTime)
+
+ if err := pe.putArrayLength(len(d.FilterResponses)); err != nil {
+ return err
+ }
+
+ for _, filterResponse := range d.FilterResponses {
+ if err := filterResponse.encode(pe, d.Version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (d *DeleteAclsResponse) decode(pd packetDecoder, version int16) (err error) {
+ if d.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ d.FilterResponses = make([]*FilterResponse, n)
+
+ for i := 0; i < n; i++ {
+ d.FilterResponses[i] = new(FilterResponse)
+ if err := d.FilterResponses[i].decode(pd, version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (d *DeleteAclsResponse) key() int16 {
+ return apiKeyDeleteAcls
+}
+
+func (d *DeleteAclsResponse) version() int16 {
+ return d.Version
+}
+
+func (d *DeleteAclsResponse) headerVersion() int16 {
+ return 0
+}
+
+func (d *DeleteAclsResponse) isValidVersion() bool {
+ return d.Version >= 0 && d.Version <= 1
+}
+
+func (d *DeleteAclsResponse) requiredVersion() KafkaVersion {
+ switch d.Version {
+ case 1:
+ return V2_0_0_0
+ default:
+ return V0_11_0_0
+ }
+}
+
+func (r *DeleteAclsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
+
+// FilterResponse is a filter response type
+type FilterResponse struct {
+ Err KError
+ ErrMsg *string
+ MatchingAcls []*MatchingAcl
+}
+
+func (f *FilterResponse) encode(pe packetEncoder, version int16) error {
+ pe.putKError(f.Err)
+ if err := pe.putNullableString(f.ErrMsg); err != nil {
+ return err
+ }
+
+ if err := pe.putArrayLength(len(f.MatchingAcls)); err != nil {
+ return err
+ }
+ for _, matchingAcl := range f.MatchingAcls {
+ if err := matchingAcl.encode(pe, version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (f *FilterResponse) decode(pd packetDecoder, version int16) (err error) {
+ f.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if f.ErrMsg, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ f.MatchingAcls = make([]*MatchingAcl, n)
+ for i := 0; i < n; i++ {
+ f.MatchingAcls[i] = new(MatchingAcl)
+ if err := f.MatchingAcls[i].decode(pd, version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// MatchingAcl is a matching acl type
+type MatchingAcl struct {
+ Err KError
+ ErrMsg *string
+ Resource
+ Acl
+}
+
+func (m *MatchingAcl) encode(pe packetEncoder, version int16) error {
+ pe.putKError(m.Err)
+ if err := pe.putNullableString(m.ErrMsg); err != nil {
+ return err
+ }
+
+ if err := m.Resource.encode(pe, version); err != nil {
+ return err
+ }
+
+ if err := m.Acl.encode(pe); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (m *MatchingAcl) decode(pd packetDecoder, version int16) (err error) {
+ m.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if m.ErrMsg, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ if err := m.Resource.decode(pd, version); err != nil {
+ return err
+ }
+
+ if err := m.Acl.decode(pd, version); err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/acl_describe_request.go b/vendor/github.com/IBM/sarama/acl_describe_request.go
new file mode 100644
index 0000000..6eb218f
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/acl_describe_request.go
@@ -0,0 +1,47 @@
+package sarama
+
+// DescribeAclsRequest is a describe acl request type
+type DescribeAclsRequest struct {
+ Version int
+ AclFilter
+}
+
+func (d *DescribeAclsRequest) setVersion(v int16) {
+ d.Version = int(v)
+}
+
+func (d *DescribeAclsRequest) encode(pe packetEncoder) error {
+ d.AclFilter.Version = d.Version
+ return d.AclFilter.encode(pe)
+}
+
+func (d *DescribeAclsRequest) decode(pd packetDecoder, version int16) (err error) {
+ d.Version = int(version)
+ d.AclFilter.Version = int(version)
+ return d.AclFilter.decode(pd, version)
+}
+
+func (d *DescribeAclsRequest) key() int16 {
+ return apiKeyDescribeAcls
+}
+
+func (d *DescribeAclsRequest) version() int16 {
+ return int16(d.Version)
+}
+
+func (d *DescribeAclsRequest) headerVersion() int16 {
+ return 1
+}
+
+func (d *DescribeAclsRequest) isValidVersion() bool {
+ return d.Version >= 0 && d.Version <= 1
+}
+
+func (d *DescribeAclsRequest) requiredVersion() KafkaVersion {
+ switch d.Version {
+ case 1:
+ return V2_0_0_0
+ default:
+ return V0_11_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/acl_describe_response.go b/vendor/github.com/IBM/sarama/acl_describe_response.go
new file mode 100644
index 0000000..c9e2552
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/acl_describe_response.go
@@ -0,0 +1,100 @@
+package sarama
+
+import "time"
+
+// DescribeAclsResponse is a describe acl response type
+type DescribeAclsResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ Err KError
+ ErrMsg *string
+ ResourceAcls []*ResourceAcls
+}
+
+func (d *DescribeAclsResponse) setVersion(v int16) {
+ d.Version = v
+}
+
+func (d *DescribeAclsResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(d.ThrottleTime)
+ pe.putKError(d.Err)
+
+ if err := pe.putNullableString(d.ErrMsg); err != nil {
+ return err
+ }
+
+ if err := pe.putArrayLength(len(d.ResourceAcls)); err != nil {
+ return err
+ }
+
+ for _, resourceAcl := range d.ResourceAcls {
+ if err := resourceAcl.encode(pe, d.Version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (d *DescribeAclsResponse) decode(pd packetDecoder, version int16) (err error) {
+ if d.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ d.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ errmsg, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ if errmsg != "" {
+ d.ErrMsg = &errmsg
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ d.ResourceAcls = make([]*ResourceAcls, n)
+
+ for i := 0; i < n; i++ {
+ d.ResourceAcls[i] = new(ResourceAcls)
+ if err := d.ResourceAcls[i].decode(pd, version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (d *DescribeAclsResponse) key() int16 {
+ return apiKeyDescribeAcls
+}
+
+func (d *DescribeAclsResponse) version() int16 {
+ return d.Version
+}
+
+func (d *DescribeAclsResponse) headerVersion() int16 {
+ return 0
+}
+
+func (d *DescribeAclsResponse) isValidVersion() bool {
+ return d.Version >= 0 && d.Version <= 1
+}
+
+func (d *DescribeAclsResponse) requiredVersion() KafkaVersion {
+ switch d.Version {
+ case 1:
+ return V2_0_0_0
+ default:
+ return V0_11_0_0
+ }
+}
+
+func (r *DescribeAclsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/acl_filter.go b/vendor/github.com/IBM/sarama/acl_filter.go
new file mode 100644
index 0000000..b380161
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/acl_filter.go
@@ -0,0 +1,77 @@
+package sarama
+
+type AclFilter struct {
+ Version int
+ ResourceType AclResourceType
+ ResourceName *string
+ ResourcePatternTypeFilter AclResourcePatternType
+ Principal *string
+ Host *string
+ Operation AclOperation
+ PermissionType AclPermissionType
+}
+
+func (a *AclFilter) encode(pe packetEncoder) error {
+ pe.putInt8(int8(a.ResourceType))
+ if err := pe.putNullableString(a.ResourceName); err != nil {
+ return err
+ }
+
+ if a.Version == 1 {
+ pe.putInt8(int8(a.ResourcePatternTypeFilter))
+ }
+
+ if err := pe.putNullableString(a.Principal); err != nil {
+ return err
+ }
+ if err := pe.putNullableString(a.Host); err != nil {
+ return err
+ }
+ pe.putInt8(int8(a.Operation))
+ pe.putInt8(int8(a.PermissionType))
+
+ return nil
+}
+
+func (a *AclFilter) decode(pd packetDecoder, version int16) (err error) {
+ resourceType, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ a.ResourceType = AclResourceType(resourceType)
+
+ if a.ResourceName, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ if a.Version == 1 {
+ pattern, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+
+ a.ResourcePatternTypeFilter = AclResourcePatternType(pattern)
+ }
+
+ if a.Principal, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ if a.Host, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ operation, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ a.Operation = AclOperation(operation)
+
+ permissionType, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ a.PermissionType = AclPermissionType(permissionType)
+
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/acl_types.go b/vendor/github.com/IBM/sarama/acl_types.go
new file mode 100644
index 0000000..62bb534
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/acl_types.go
@@ -0,0 +1,238 @@
+package sarama
+
+import (
+ "fmt"
+ "strings"
+)
+
+type (
+ AclOperation int
+
+ AclPermissionType int
+
+ AclResourceType int
+
+ AclResourcePatternType int
+)
+
+// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclOperation.java
+const (
+ AclOperationUnknown AclOperation = iota
+ AclOperationAny
+ AclOperationAll
+ AclOperationRead
+ AclOperationWrite
+ AclOperationCreate
+ AclOperationDelete
+ AclOperationAlter
+ AclOperationDescribe
+ AclOperationClusterAction
+ AclOperationDescribeConfigs
+ AclOperationAlterConfigs
+ AclOperationIdempotentWrite
+)
+
+func (a *AclOperation) String() string {
+ mapping := map[AclOperation]string{
+ AclOperationUnknown: "Unknown",
+ AclOperationAny: "Any",
+ AclOperationAll: "All",
+ AclOperationRead: "Read",
+ AclOperationWrite: "Write",
+ AclOperationCreate: "Create",
+ AclOperationDelete: "Delete",
+ AclOperationAlter: "Alter",
+ AclOperationDescribe: "Describe",
+ AclOperationClusterAction: "ClusterAction",
+ AclOperationDescribeConfigs: "DescribeConfigs",
+ AclOperationAlterConfigs: "AlterConfigs",
+ AclOperationIdempotentWrite: "IdempotentWrite",
+ }
+ s, ok := mapping[*a]
+ if !ok {
+ s = mapping[AclOperationUnknown]
+ }
+ return s
+}
+
+// MarshalText returns the text form of the AclOperation (name without prefix)
+func (a *AclOperation) MarshalText() ([]byte, error) {
+ return []byte(a.String()), nil
+}
+
+// UnmarshalText takes a text representation of the operation and converts it to an AclOperation
+func (a *AclOperation) UnmarshalText(text []byte) error {
+ normalized := strings.ToLower(string(text))
+ mapping := map[string]AclOperation{
+ "unknown": AclOperationUnknown,
+ "any": AclOperationAny,
+ "all": AclOperationAll,
+ "read": AclOperationRead,
+ "write": AclOperationWrite,
+ "create": AclOperationCreate,
+ "delete": AclOperationDelete,
+ "alter": AclOperationAlter,
+ "describe": AclOperationDescribe,
+ "clusteraction": AclOperationClusterAction,
+ "describeconfigs": AclOperationDescribeConfigs,
+ "alterconfigs": AclOperationAlterConfigs,
+ "idempotentwrite": AclOperationIdempotentWrite,
+ }
+ ao, ok := mapping[normalized]
+ if !ok {
+ *a = AclOperationUnknown
+ return fmt.Errorf("no acl operation with name %s", normalized)
+ }
+ *a = ao
+ return nil
+}
+
+// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclPermissionType.java
+const (
+ AclPermissionUnknown AclPermissionType = iota
+ AclPermissionAny
+ AclPermissionDeny
+ AclPermissionAllow
+)
+
+func (a *AclPermissionType) String() string {
+ mapping := map[AclPermissionType]string{
+ AclPermissionUnknown: "Unknown",
+ AclPermissionAny: "Any",
+ AclPermissionDeny: "Deny",
+ AclPermissionAllow: "Allow",
+ }
+ s, ok := mapping[*a]
+ if !ok {
+ s = mapping[AclPermissionUnknown]
+ }
+ return s
+}
+
+// MarshalText returns the text form of the AclPermissionType (name without prefix)
+func (a *AclPermissionType) MarshalText() ([]byte, error) {
+ return []byte(a.String()), nil
+}
+
+// UnmarshalText takes a text representation of the permission type and converts it to an AclPermissionType
+func (a *AclPermissionType) UnmarshalText(text []byte) error {
+ normalized := strings.ToLower(string(text))
+ mapping := map[string]AclPermissionType{
+ "unknown": AclPermissionUnknown,
+ "any": AclPermissionAny,
+ "deny": AclPermissionDeny,
+ "allow": AclPermissionAllow,
+ }
+
+ apt, ok := mapping[normalized]
+ if !ok {
+ *a = AclPermissionUnknown
+ return fmt.Errorf("no acl permission with name %s", normalized)
+ }
+ *a = apt
+ return nil
+}
+
+// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java
+const (
+ AclResourceUnknown AclResourceType = iota
+ AclResourceAny
+ AclResourceTopic
+ AclResourceGroup
+ AclResourceCluster
+ AclResourceTransactionalID
+ AclResourceDelegationToken
+)
+
+func (a *AclResourceType) String() string {
+ mapping := map[AclResourceType]string{
+ AclResourceUnknown: "Unknown",
+ AclResourceAny: "Any",
+ AclResourceTopic: "Topic",
+ AclResourceGroup: "Group",
+ AclResourceCluster: "Cluster",
+ AclResourceTransactionalID: "TransactionalID",
+ AclResourceDelegationToken: "DelegationToken",
+ }
+ s, ok := mapping[*a]
+ if !ok {
+ s = mapping[AclResourceUnknown]
+ }
+ return s
+}
+
+// MarshalText returns the text form of the AclResourceType (name without prefix)
+func (a *AclResourceType) MarshalText() ([]byte, error) {
+ return []byte(a.String()), nil
+}
+
+// UnmarshalText takes a text representation of the resource type and converts it to an AclResourceType
+func (a *AclResourceType) UnmarshalText(text []byte) error {
+ normalized := strings.ToLower(string(text))
+ mapping := map[string]AclResourceType{
+ "unknown": AclResourceUnknown,
+ "any": AclResourceAny,
+ "topic": AclResourceTopic,
+ "group": AclResourceGroup,
+ "cluster": AclResourceCluster,
+ "transactionalid": AclResourceTransactionalID,
+ "delegationtoken": AclResourceDelegationToken,
+ }
+
+ art, ok := mapping[normalized]
+ if !ok {
+ *a = AclResourceUnknown
+ return fmt.Errorf("no acl resource with name %s", normalized)
+ }
+ *a = art
+ return nil
+}
+
+// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/PatternType.java
+const (
+ AclPatternUnknown AclResourcePatternType = iota
+ AclPatternAny
+ AclPatternMatch
+ AclPatternLiteral
+ AclPatternPrefixed
+)
+
+func (a *AclResourcePatternType) String() string {
+ mapping := map[AclResourcePatternType]string{
+ AclPatternUnknown: "Unknown",
+ AclPatternAny: "Any",
+ AclPatternMatch: "Match",
+ AclPatternLiteral: "Literal",
+ AclPatternPrefixed: "Prefixed",
+ }
+ s, ok := mapping[*a]
+ if !ok {
+ s = mapping[AclPatternUnknown]
+ }
+ return s
+}
+
+// MarshalText returns the text form of the AclResourcePatternType (name without prefix)
+func (a *AclResourcePatternType) MarshalText() ([]byte, error) {
+ return []byte(a.String()), nil
+}
+
+// UnmarshalText takes a text representation of the resource pattern type and converts it to an AclResourcePatternType
+func (a *AclResourcePatternType) UnmarshalText(text []byte) error {
+ normalized := strings.ToLower(string(text))
+ mapping := map[string]AclResourcePatternType{
+ "unknown": AclPatternUnknown,
+ "any": AclPatternAny,
+ "match": AclPatternMatch,
+ "literal": AclPatternLiteral,
+ "prefixed": AclPatternPrefixed,
+ }
+
+ arpt, ok := mapping[normalized]
+ if !ok {
+ *a = AclPatternUnknown
+ return fmt.Errorf("no acl resource pattern with name %s", normalized)
+ }
+ *a = arpt
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/add_offsets_to_txn_request.go b/vendor/github.com/IBM/sarama/add_offsets_to_txn_request.go
new file mode 100644
index 0000000..4361abb
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/add_offsets_to_txn_request.go
@@ -0,0 +1,75 @@
+package sarama
+
+// AddOffsetsToTxnRequest adds offsets to a transaction request
+type AddOffsetsToTxnRequest struct {
+ Version int16
+ TransactionalID string
+ ProducerID int64
+ ProducerEpoch int16
+ GroupID string
+}
+
+func (a *AddOffsetsToTxnRequest) setVersion(v int16) {
+ a.Version = v
+}
+
+func (a *AddOffsetsToTxnRequest) encode(pe packetEncoder) error {
+ if err := pe.putString(a.TransactionalID); err != nil {
+ return err
+ }
+
+ pe.putInt64(a.ProducerID)
+
+ pe.putInt16(a.ProducerEpoch)
+
+ if err := pe.putString(a.GroupID); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (a *AddOffsetsToTxnRequest) decode(pd packetDecoder, version int16) (err error) {
+ if a.TransactionalID, err = pd.getString(); err != nil {
+ return err
+ }
+ if a.ProducerID, err = pd.getInt64(); err != nil {
+ return err
+ }
+ if a.ProducerEpoch, err = pd.getInt16(); err != nil {
+ return err
+ }
+ if a.GroupID, err = pd.getString(); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (a *AddOffsetsToTxnRequest) key() int16 {
+ return apiKeyAddOffsetsToTxn
+}
+
+func (a *AddOffsetsToTxnRequest) version() int16 {
+ return a.Version
+}
+
+func (a *AddOffsetsToTxnRequest) headerVersion() int16 {
+ return 1
+}
+
+func (a *AddOffsetsToTxnRequest) isValidVersion() bool {
+ return a.Version >= 0 && a.Version <= 2
+}
+
+func (a *AddOffsetsToTxnRequest) requiredVersion() KafkaVersion {
+ switch a.Version {
+ case 2:
+ return V2_7_0_0
+ case 1:
+ return V2_0_0_0
+ case 0:
+ return V0_11_0_0
+ default:
+ return V2_7_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/add_offsets_to_txn_response.go b/vendor/github.com/IBM/sarama/add_offsets_to_txn_response.go
new file mode 100644
index 0000000..17858a1
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/add_offsets_to_txn_response.go
@@ -0,0 +1,68 @@
+package sarama
+
+import (
+ "time"
+)
+
+// AddOffsetsToTxnResponse is a response type for adding offsets to txns
+type AddOffsetsToTxnResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ Err KError
+}
+
+func (a *AddOffsetsToTxnResponse) setVersion(v int16) {
+ a.Version = v
+}
+
+func (a *AddOffsetsToTxnResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(a.ThrottleTime)
+ pe.putKError(a.Err)
+ return nil
+}
+
+func (a *AddOffsetsToTxnResponse) decode(pd packetDecoder, version int16) (err error) {
+ if a.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ a.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (a *AddOffsetsToTxnResponse) key() int16 {
+ return apiKeyAddOffsetsToTxn
+}
+
+func (a *AddOffsetsToTxnResponse) version() int16 {
+ return a.Version
+}
+
+func (a *AddOffsetsToTxnResponse) headerVersion() int16 {
+ return 0
+}
+
+func (a *AddOffsetsToTxnResponse) isValidVersion() bool {
+ return a.Version >= 0 && a.Version <= 2
+}
+
+func (a *AddOffsetsToTxnResponse) requiredVersion() KafkaVersion {
+ switch a.Version {
+ case 2:
+ return V2_7_0_0
+ case 1:
+ return V2_0_0_0
+ case 0:
+ return V0_11_0_0
+ default:
+ return V2_7_0_0
+ }
+}
+
+func (r *AddOffsetsToTxnResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/add_partitions_to_txn_request.go b/vendor/github.com/IBM/sarama/add_partitions_to_txn_request.go
new file mode 100644
index 0000000..492023b
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/add_partitions_to_txn_request.go
@@ -0,0 +1,97 @@
+package sarama
+
+// AddPartitionsToTxnRequest is a add partition request
+type AddPartitionsToTxnRequest struct {
+ Version int16
+ TransactionalID string
+ ProducerID int64
+ ProducerEpoch int16
+ TopicPartitions map[string][]int32
+}
+
+func (a *AddPartitionsToTxnRequest) setVersion(v int16) {
+ a.Version = v
+}
+
+func (a *AddPartitionsToTxnRequest) encode(pe packetEncoder) error {
+ if err := pe.putString(a.TransactionalID); err != nil {
+ return err
+ }
+ pe.putInt64(a.ProducerID)
+ pe.putInt16(a.ProducerEpoch)
+
+ if err := pe.putArrayLength(len(a.TopicPartitions)); err != nil {
+ return err
+ }
+ for topic, partitions := range a.TopicPartitions {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putInt32Array(partitions); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (a *AddPartitionsToTxnRequest) decode(pd packetDecoder, version int16) (err error) {
+ if a.TransactionalID, err = pd.getString(); err != nil {
+ return err
+ }
+ if a.ProducerID, err = pd.getInt64(); err != nil {
+ return err
+ }
+ if a.ProducerEpoch, err = pd.getInt16(); err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ a.TopicPartitions = make(map[string][]int32)
+ for i := 0; i < n; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ partitions, err := pd.getInt32Array()
+ if err != nil {
+ return err
+ }
+
+ a.TopicPartitions[topic] = partitions
+ }
+
+ return nil
+}
+
+func (a *AddPartitionsToTxnRequest) key() int16 {
+ return apiKeyAddPartitionsToTxn
+}
+
+func (a *AddPartitionsToTxnRequest) version() int16 {
+ return a.Version
+}
+
+func (a *AddPartitionsToTxnRequest) headerVersion() int16 {
+ return 1
+}
+
+func (a *AddPartitionsToTxnRequest) isValidVersion() bool {
+ return a.Version >= 0 && a.Version <= 2
+}
+
+func (a *AddPartitionsToTxnRequest) requiredVersion() KafkaVersion {
+ switch a.Version {
+ case 2:
+ return V2_7_0_0
+ case 1:
+ return V2_0_0_0
+ default:
+ return V0_11_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/add_partitions_to_txn_response.go b/vendor/github.com/IBM/sarama/add_partitions_to_txn_response.go
new file mode 100644
index 0000000..0384263
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/add_partitions_to_txn_response.go
@@ -0,0 +1,132 @@
+package sarama
+
+import (
+ "time"
+)
+
+// AddPartitionsToTxnResponse is a partition errors to transaction type
+type AddPartitionsToTxnResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ Errors map[string][]*PartitionError
+}
+
+func (a *AddPartitionsToTxnResponse) setVersion(v int16) {
+ a.Version = v
+}
+
+func (a *AddPartitionsToTxnResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(a.ThrottleTime)
+ if err := pe.putArrayLength(len(a.Errors)); err != nil {
+ return err
+ }
+
+ for topic, e := range a.Errors {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putArrayLength(len(e)); err != nil {
+ return err
+ }
+ for _, partitionError := range e {
+ if err := partitionError.encode(pe); err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func (a *AddPartitionsToTxnResponse) decode(pd packetDecoder, version int16) (err error) {
+ a.Version = version
+ if a.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ a.Errors = make(map[string][]*PartitionError)
+
+ for i := 0; i < n; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ m, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ a.Errors[topic] = make([]*PartitionError, m)
+
+ for j := 0; j < m; j++ {
+ a.Errors[topic][j] = new(PartitionError)
+ if err := a.Errors[topic][j].decode(pd, version); err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func (a *AddPartitionsToTxnResponse) key() int16 {
+ return apiKeyAddPartitionsToTxn
+}
+
+func (a *AddPartitionsToTxnResponse) version() int16 {
+ return a.Version
+}
+
+func (a *AddPartitionsToTxnResponse) headerVersion() int16 {
+ return 0
+}
+
+func (a *AddPartitionsToTxnResponse) isValidVersion() bool {
+ return a.Version >= 0 && a.Version <= 2
+}
+
+func (a *AddPartitionsToTxnResponse) requiredVersion() KafkaVersion {
+ switch a.Version {
+ case 2:
+ return V2_7_0_0
+ case 1:
+ return V2_0_0_0
+ default:
+ return V0_11_0_0
+ }
+}
+
+func (r *AddPartitionsToTxnResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
+
+// PartitionError is a partition error type
+type PartitionError struct {
+ Partition int32
+ Err KError
+}
+
+func (p *PartitionError) encode(pe packetEncoder) error {
+ pe.putInt32(p.Partition)
+ pe.putKError(p.Err)
+ return nil
+}
+
+func (p *PartitionError) decode(pd packetDecoder, version int16) (err error) {
+ if p.Partition, err = pd.getInt32(); err != nil {
+ return err
+ }
+
+ p.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/admin.go b/vendor/github.com/IBM/sarama/admin.go
new file mode 100644
index 0000000..dda0201
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/admin.go
@@ -0,0 +1,1395 @@
+package sarama
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "maps"
+ "math/rand"
+ "strconv"
+ "sync"
+ "time"
+)
+
+// ClusterAdmin is the administrative client for Kafka, which supports managing and inspecting topics,
+// brokers, configurations and ACLs. The minimum broker version required is 0.10.0.0.
+// Methods with stricter requirements will specify the minimum broker version required.
+// You MUST call Close() on a client to avoid leaks
+type ClusterAdmin interface {
+ // Creates a new topic. This operation is supported by brokers with version 0.10.1.0 or higher.
+ // It may take several seconds after CreateTopic returns success for all the brokers
+ // to become aware that the topic has been created. During this time, listTopics
+ // may not return information about the new topic.The validateOnly option is supported from version 0.10.2.0.
+ CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error
+
+ // List the topics available in the cluster with the default options.
+ ListTopics() (map[string]TopicDetail, error)
+
+ // Describe some topics in the cluster.
+ DescribeTopics(topics []string) (metadata []*TopicMetadata, err error)
+
+ // Delete a topic. It may take several seconds after the DeleteTopic to returns success
+ // and for all the brokers to become aware that the topics are gone.
+ // During this time, listTopics may continue to return information about the deleted topic.
+ // If delete.topic.enable is false on the brokers, deleteTopic will mark
+ // the topic for deletion, but not actually delete them.
+ // This operation is supported by brokers with version 0.10.1.0 or higher.
+ DeleteTopic(topic string) error
+
+ // Increase the number of partitions of the topics according to the corresponding values.
+ // If partitions are increased for a topic that has a key, the partition logic or ordering of
+ // the messages will be affected. It may take several seconds after this method returns
+ // success for all the brokers to become aware that the partitions have been created.
+ // During this time, ClusterAdmin#describeTopics may not return information about the
+ // new partitions. This operation is supported by brokers with version 1.0.0 or higher.
+ CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error
+
+ // Alter the replica assignment for partitions.
+ // This operation is supported by brokers with version 2.4.0.0 or higher.
+ AlterPartitionReassignments(topic string, assignment [][]int32) error
+
+ // Provides info on ongoing partitions replica reassignments.
+ // This operation is supported by brokers with version 2.4.0.0 or higher.
+ ListPartitionReassignments(topics string, partitions []int32) (topicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus, err error)
+
+ // Delete records whose offset is smaller than the given offset of the corresponding partition.
+ // This operation is supported by brokers with version 0.11.0.0 or higher.
+ DeleteRecords(topic string, partitionOffsets map[int32]int64) error
+
+ // Get the configuration for the specified resources.
+ // The returned configuration includes default values and the Default is true
+ // can be used to distinguish them from user supplied values.
+ // Config entries where ReadOnly is true cannot be updated.
+ // The value of config entries where Sensitive is true is always nil so
+ // sensitive information is not disclosed.
+ // This operation is supported by brokers with version 0.11.0.0 or higher.
+ DescribeConfig(resource ConfigResource) ([]ConfigEntry, error)
+
+ // Update the configuration for the specified resources with the default options.
+ // This operation is supported by brokers with version 0.11.0.0 or higher.
+ // The resources with their configs (topic is the only resource type with configs
+ // that can be updated currently Updates are not transactional so they may succeed
+ // for some resources while fail for others. The configs for a particular resource are updated automatically.
+ AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error
+
+ // IncrementalAlterConfig Incrementally Update the configuration for the specified resources with the default options.
+ // This operation is supported by brokers with version 2.3.0.0 or higher.
+ // Updates are not transactional so they may succeed for some resources while fail for others.
+ // The configs for a particular resource are updated automatically.
+ IncrementalAlterConfig(resourceType ConfigResourceType, name string, entries map[string]IncrementalAlterConfigsEntry, validateOnly bool) error
+
+ // Creates an access control list (ACL) which is bound to a specific resource.
+ // This operation is not transactional so it may succeed or fail.
+ // If you attempt to add an ACL that duplicates an existing ACL, no error will be raised, but
+ // no changes will be made. This operation is supported by brokers with version 0.11.0.0 or higher.
+ // Deprecated: Use CreateACLs instead.
+ CreateACL(resource Resource, acl Acl) error
+
+ // Creates access control lists (ACLs) which are bound to specific resources.
+ // This operation is not transactional so it may succeed for some ACLs while fail for others.
+ // If you attempt to add an ACL that duplicates an existing ACL, no error will be raised, but
+ // no changes will be made. This operation is supported by brokers with version 0.11.0.0 or higher.
+ CreateACLs([]*ResourceAcls) error
+
+ // Lists access control lists (ACLs) according to the supplied filter.
+ // it may take some time for changes made by createAcls or deleteAcls to be reflected in the output of ListAcls
+ // This operation is supported by brokers with version 0.11.0.0 or higher.
+ ListAcls(filter AclFilter) ([]ResourceAcls, error)
+
+ // Deletes access control lists (ACLs) according to the supplied filters.
+ // This operation is not transactional so it may succeed for some ACLs while fail for others.
+ // This operation is supported by brokers with version 0.11.0.0 or higher.
+ DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error)
+
+ // ElectLeaders allows to trigger the election of preferred leaders for a set of partitions.
+ ElectLeaders(ElectionType, map[string][]int32) (map[string]map[int32]*PartitionResult, error)
+
+ // List the consumer groups available in the cluster.
+ ListConsumerGroups() (map[string]string, error)
+
+ // Describe the given consumer groups.
+ DescribeConsumerGroups(groups []string) ([]*GroupDescription, error)
+
+ // List the consumer group offsets available in the cluster.
+ ListConsumerGroupOffsets(group string, topicPartitions map[string][]int32) (*OffsetFetchResponse, error)
+
+ // Deletes a consumer group offset
+ DeleteConsumerGroupOffset(group string, topic string, partition int32) error
+
+ // Delete a consumer group.
+ DeleteConsumerGroup(group string) error
+
+ // Get information about the nodes in the cluster
+ DescribeCluster() (brokers []*Broker, controllerID int32, err error)
+
+ // Get information about all log directories on the given set of brokers
+ DescribeLogDirs(brokers []int32) (map[int32][]DescribeLogDirsResponseDirMetadata, error)
+
+ // Get information about SCRAM users
+ DescribeUserScramCredentials(users []string) ([]*DescribeUserScramCredentialsResult, error)
+
+ // Delete SCRAM users
+ DeleteUserScramCredentials(delete []AlterUserScramCredentialsDelete) ([]*AlterUserScramCredentialsResult, error)
+
+ // Upsert SCRAM users
+ UpsertUserScramCredentials(upsert []AlterUserScramCredentialsUpsert) ([]*AlterUserScramCredentialsResult, error)
+
+ // Get client quota configurations corresponding to the specified filter.
+ // This operation is supported by brokers with version 2.6.0.0 or higher.
+ DescribeClientQuotas(components []QuotaFilterComponent, strict bool) ([]DescribeClientQuotasEntry, error)
+
+ // Alters client quota configurations with the specified alterations.
+ // This operation is supported by brokers with version 2.6.0.0 or higher.
+ AlterClientQuotas(entity []QuotaEntityComponent, op ClientQuotasOp, validateOnly bool) error
+
+ // Controller returns the cluster controller broker. It will return a
+ // locally cached value if it's available.
+ Controller() (*Broker, error)
+
+ // Coordinator returns the coordinating broker for a consumer group. It will
+ // return a locally cached value if it's available.
+ Coordinator(group string) (*Broker, error)
+
+ // Remove members from the consumer group by given member identities.
+ // This operation is supported by brokers with version 2.3 or higher
+ // This is for static membership feature. KIP-345
+ RemoveMemberFromConsumerGroup(groupId string, groupInstanceIds []string) (*LeaveGroupResponse, error)
+
+ // Close shuts down the admin and closes underlying client.
+ Close() error
+}
+
+type clusterAdmin struct {
+ client Client
+ conf *Config
+}
+
+// NewClusterAdmin creates a new ClusterAdmin using the given broker addresses and configuration.
+func NewClusterAdmin(addrs []string, conf *Config) (ClusterAdmin, error) {
+ client, err := NewClient(addrs, conf)
+ if err != nil {
+ return nil, err
+ }
+ admin, err := NewClusterAdminFromClient(client)
+ if err != nil {
+ client.Close()
+ }
+ return admin, err
+}
+
+// NewClusterAdminFromClient creates a new ClusterAdmin using the given client.
+// Note that underlying client will also be closed on admin's Close() call.
+func NewClusterAdminFromClient(client Client) (ClusterAdmin, error) {
+ // make sure we can retrieve the controller
+ _, err := client.Controller()
+ if err != nil {
+ return nil, err
+ }
+
+ ca := &clusterAdmin{
+ client: client,
+ conf: client.Config(),
+ }
+ return ca, nil
+}
+
+func (ca *clusterAdmin) Close() error {
+ return ca.client.Close()
+}
+
+func (ca *clusterAdmin) Controller() (*Broker, error) {
+ return ca.client.Controller()
+}
+
+func (ca *clusterAdmin) Coordinator(group string) (*Broker, error) {
+ return ca.client.Coordinator(group)
+}
+
+func (ca *clusterAdmin) refreshController() (*Broker, error) {
+ return ca.client.RefreshController()
+}
+
+// isRetriableControllerError returns `true` if the given error type unwraps to
+// an `ErrNotController` or `EOF` response from Kafka
+func isRetriableControllerError(err error) bool {
+ return errors.Is(err, ErrNotController) || errors.Is(err, io.EOF)
+}
+
+// isRetriableGroupCoordinatorError returns `true` if the given error type
+// unwraps to an `ErrNotCoordinatorForConsumer`,
+// `ErrConsumerCoordinatorNotAvailable` or `EOF` response from Kafka
+func isRetriableGroupCoordinatorError(err error) bool {
+ return errors.Is(err, ErrNotCoordinatorForConsumer) || errors.Is(err, ErrConsumerCoordinatorNotAvailable) || errors.Is(err, io.EOF)
+}
+
+// retryOnError will repeatedly call the given (error-returning) func in the
+// case that its response is non-nil and retryable (as determined by the
+// provided retryable func) up to the maximum number of tries permitted by
+// the admin client configuration
+func (ca *clusterAdmin) retryOnError(retryable func(error) bool, fn func() error) error {
+ for attemptsRemaining := ca.conf.Admin.Retry.Max + 1; ; {
+ err := fn()
+ attemptsRemaining--
+ if err == nil || attemptsRemaining <= 0 || !retryable(err) {
+ return err
+ }
+ Logger.Printf(
+ "admin/request retrying after %dms... (%d attempts remaining)\n",
+ ca.conf.Admin.Retry.Backoff/time.Millisecond, attemptsRemaining)
+ time.Sleep(ca.conf.Admin.Retry.Backoff)
+ }
+}
+
+func (ca *clusterAdmin) CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error {
+ if topic == "" {
+ return ErrInvalidTopic
+ }
+
+ if detail == nil {
+ return errors.New("you must specify topic details")
+ }
+
+ topicDetails := map[string]*TopicDetail{
+ topic: detail,
+ }
+
+ request := NewCreateTopicsRequest(
+ ca.conf.Version,
+ topicDetails,
+ ca.conf.Admin.Timeout,
+ validateOnly,
+ )
+
+ return ca.retryOnError(isRetriableControllerError, func() error {
+ b, err := ca.Controller()
+ if err != nil {
+ return err
+ }
+
+ rsp, err := b.CreateTopics(request)
+ if err != nil {
+ return err
+ }
+
+ topicErr, ok := rsp.TopicErrors[topic]
+ if !ok {
+ return ErrIncompleteResponse
+ }
+
+ if !errors.Is(topicErr.Err, ErrNoError) {
+ if isRetriableControllerError(topicErr.Err) {
+ _, _ = ca.refreshController()
+ }
+ return topicErr
+ }
+
+ return nil
+ })
+}
+
+func (ca *clusterAdmin) DescribeTopics(topics []string) (metadata []*TopicMetadata, err error) {
+ var response *MetadataResponse
+ err = ca.retryOnError(isRetriableControllerError, func() error {
+ controller, err := ca.Controller()
+ if err != nil {
+ return err
+ }
+ request := NewMetadataRequest(ca.conf.Version, topics)
+ response, err = controller.GetMetadata(request)
+ if isRetriableControllerError(err) {
+ _, _ = ca.refreshController()
+ }
+ return err
+ })
+ if err != nil {
+ return nil, err
+ }
+ return response.Topics, nil
+}
+
+func (ca *clusterAdmin) DescribeCluster() (brokers []*Broker, controllerID int32, err error) {
+ var response *MetadataResponse
+ err = ca.retryOnError(isRetriableControllerError, func() error {
+ controller, err := ca.Controller()
+ if err != nil {
+ return err
+ }
+
+ request := NewMetadataRequest(ca.conf.Version, nil)
+ response, err = controller.GetMetadata(request)
+ if isRetriableControllerError(err) {
+ _, _ = ca.refreshController()
+ }
+ return err
+ })
+ if err != nil {
+ return nil, int32(0), err
+ }
+
+ return response.Brokers, response.ControllerID, nil
+}
+
+func (ca *clusterAdmin) findBroker(id int32) (*Broker, error) {
+ brokers := ca.client.Brokers()
+ for _, b := range brokers {
+ if b.ID() == id {
+ return b, nil
+ }
+ }
+ return nil, fmt.Errorf("could not find broker id %d", id)
+}
+
+func (ca *clusterAdmin) findAnyBroker() (*Broker, error) {
+ brokers := ca.client.Brokers()
+ if len(brokers) > 0 {
+ index := rand.Intn(len(brokers))
+ return brokers[index], nil
+ }
+ return nil, errors.New("no available broker")
+}
+
+func (ca *clusterAdmin) ListTopics() (map[string]TopicDetail, error) {
+ // In order to build TopicDetails we need to first get the list of all
+ // topics using a MetadataRequest and then get their configs using a
+ // DescribeConfigsRequest request. To avoid sending many requests to the
+ // broker, we use a single DescribeConfigsRequest.
+
+ // Send the all-topic MetadataRequest
+ b, err := ca.findAnyBroker()
+ if err != nil {
+ return nil, err
+ }
+ _ = b.Open(ca.client.Config())
+
+ metadataReq := NewMetadataRequest(ca.conf.Version, nil)
+ metadataResp, err := b.GetMetadata(metadataReq)
+ if err != nil {
+ return nil, err
+ }
+
+ topicsDetailsMap := make(map[string]TopicDetail, len(metadataResp.Topics))
+
+ var describeConfigsResources []*ConfigResource
+
+ for _, topic := range metadataResp.Topics {
+ topicDetails := TopicDetail{
+ NumPartitions: int32(len(topic.Partitions)),
+ }
+ if len(topic.Partitions) > 0 {
+ topicDetails.ReplicaAssignment = make(map[int32][]int32, len(topic.Partitions))
+ for _, partition := range topic.Partitions {
+ topicDetails.ReplicaAssignment[partition.ID] = partition.Replicas
+ }
+ topicDetails.ReplicationFactor = int16(len(topic.Partitions[0].Replicas))
+ }
+ topicsDetailsMap[topic.Name] = topicDetails
+
+ // we populate the resources we want to describe from the MetadataResponse
+ topicResource := ConfigResource{
+ Type: TopicResource,
+ Name: topic.Name,
+ }
+ describeConfigsResources = append(describeConfigsResources, &topicResource)
+ }
+
+ // Send the DescribeConfigsRequest
+ describeConfigsReq := &DescribeConfigsRequest{
+ Resources: describeConfigsResources,
+ }
+
+ if ca.conf.Version.IsAtLeast(V1_1_0_0) {
+ describeConfigsReq.Version = 1
+ }
+
+ if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ describeConfigsReq.Version = 2
+ }
+
+ describeConfigsResp, err := b.DescribeConfigs(describeConfigsReq)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, resource := range describeConfigsResp.Resources {
+ topicDetails := topicsDetailsMap[resource.Name]
+ topicDetails.ConfigEntries = make(map[string]*string)
+
+ for _, entry := range resource.Configs {
+ // only include non-default non-sensitive config
+ // (don't actually think topic config will ever be sensitive)
+ if entry.Default || entry.Sensitive {
+ continue
+ }
+ topicDetails.ConfigEntries[entry.Name] = &entry.Value
+ }
+
+ topicsDetailsMap[resource.Name] = topicDetails
+ }
+
+ return topicsDetailsMap, nil
+}
+
+func (ca *clusterAdmin) DeleteTopic(topic string) error {
+ if topic == "" {
+ return ErrInvalidTopic
+ }
+
+ request := &DeleteTopicsRequest{
+ Topics: []string{topic},
+ Timeout: ca.conf.Admin.Timeout,
+ }
+
+ // Versions 0, 1, 2, and 3 are the same.
+ // Version 4 is first flexible version.
+ if ca.conf.Version.IsAtLeast(V2_4_0_0) {
+ request.Version = 4
+ } else if ca.conf.Version.IsAtLeast(V2_1_0_0) {
+ request.Version = 3
+ } else if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ request.Version = 2
+ } else if ca.conf.Version.IsAtLeast(V0_11_0_0) {
+ request.Version = 1
+ }
+
+ return ca.retryOnError(isRetriableControllerError, func() error {
+ b, err := ca.Controller()
+ if err != nil {
+ return err
+ }
+
+ rsp, err := b.DeleteTopics(request)
+ if err != nil {
+ return err
+ }
+
+ topicErr, ok := rsp.TopicErrorCodes[topic]
+ if !ok {
+ return ErrIncompleteResponse
+ }
+
+ if !errors.Is(topicErr, ErrNoError) {
+ if errors.Is(topicErr, ErrNotController) {
+ _, _ = ca.refreshController()
+ }
+ return topicErr
+ }
+
+ return nil
+ })
+}
+
+func (ca *clusterAdmin) CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error {
+ if topic == "" {
+ return ErrInvalidTopic
+ }
+
+ topicPartitions := map[string]*TopicPartition{
+ topic: {
+ Count: count,
+ Assignment: assignment,
+ },
+ }
+
+ request := &CreatePartitionsRequest{
+ TopicPartitions: topicPartitions,
+ Timeout: ca.conf.Admin.Timeout,
+ ValidateOnly: validateOnly,
+ }
+ if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ request.Version = 1
+ }
+
+ return ca.retryOnError(isRetriableControllerError, func() error {
+ b, err := ca.Controller()
+ if err != nil {
+ return err
+ }
+
+ rsp, err := b.CreatePartitions(request)
+ if err != nil {
+ return err
+ }
+
+ topicErr, ok := rsp.TopicPartitionErrors[topic]
+ if !ok {
+ return ErrIncompleteResponse
+ }
+
+ if !errors.Is(topicErr.Err, ErrNoError) {
+ if errors.Is(topicErr.Err, ErrNotController) {
+ _, _ = ca.refreshController()
+ }
+ return topicErr
+ }
+
+ return nil
+ })
+}
+
+func (ca *clusterAdmin) AlterPartitionReassignments(topic string, assignment [][]int32) error {
+ if topic == "" {
+ return ErrInvalidTopic
+ }
+
+ request := &AlterPartitionReassignmentsRequest{
+ TimeoutMs: int32(60000),
+ Version: int16(0),
+ }
+
+ for i := 0; i < len(assignment); i++ {
+ request.AddBlock(topic, int32(i), assignment[i])
+ }
+
+ return ca.retryOnError(isRetriableControllerError, func() error {
+ b, err := ca.Controller()
+ if err != nil {
+ return err
+ }
+
+ errs := make([]error, 0)
+
+ rsp, err := b.AlterPartitionReassignments(request)
+
+ if err != nil {
+ errs = append(errs, err)
+ } else {
+ if rsp.ErrorCode > 0 {
+ errs = append(errs, rsp.ErrorCode)
+ }
+
+ for topic, topicErrors := range rsp.Errors {
+ for partition, partitionError := range topicErrors {
+ if !errors.Is(partitionError.errorCode, ErrNoError) {
+ errs = append(errs, fmt.Errorf("[%s-%d]: %w", topic, partition, partitionError.errorCode))
+ }
+ }
+ }
+ }
+
+ if len(errs) > 0 {
+ return Wrap(ErrReassignPartitions, errs...)
+ }
+
+ return nil
+ })
+}
+
+func (ca *clusterAdmin) ListPartitionReassignments(topic string, partitions []int32) (topicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus, err error) {
+ if topic == "" {
+ return nil, ErrInvalidTopic
+ }
+
+ request := &ListPartitionReassignmentsRequest{
+ TimeoutMs: int32(60000),
+ Version: int16(0),
+ }
+
+ request.AddBlock(topic, partitions)
+
+ var rsp *ListPartitionReassignmentsResponse
+ err = ca.retryOnError(isRetriableControllerError, func() error {
+ b, err := ca.Controller()
+ if err != nil {
+ return err
+ }
+ _ = b.Open(ca.client.Config())
+
+ rsp, err = b.ListPartitionReassignments(request)
+ if isRetriableControllerError(err) {
+ _, _ = ca.refreshController()
+ }
+ return err
+ })
+
+ if err == nil && rsp != nil {
+ return rsp.TopicStatus, nil
+ } else {
+ return nil, err
+ }
+}
+
+func (ca *clusterAdmin) DeleteRecords(topic string, partitionOffsets map[int32]int64) error {
+ if topic == "" {
+ return ErrInvalidTopic
+ }
+ errs := make([]error, 0)
+ partitionPerBroker := make(map[*Broker][]int32)
+ for partition := range partitionOffsets {
+ broker, err := ca.client.Leader(topic, partition)
+ if err != nil {
+ errs = append(errs, err)
+ continue
+ }
+ partitionPerBroker[broker] = append(partitionPerBroker[broker], partition)
+ }
+ for broker, partitions := range partitionPerBroker {
+ recordsToDelete := make(map[int32]int64, len(partitions))
+ for _, p := range partitions {
+ recordsToDelete[p] = partitionOffsets[p]
+ }
+ topics := map[string]*DeleteRecordsRequestTopic{
+ topic: {
+ PartitionOffsets: recordsToDelete,
+ },
+ }
+ request := &DeleteRecordsRequest{
+ Topics: topics,
+ Timeout: ca.conf.Admin.Timeout,
+ }
+ if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ request.Version = 1
+ }
+ rsp, err := broker.DeleteRecords(request)
+ if err != nil {
+ errs = append(errs, err)
+ continue
+ }
+
+ deleteRecordsResponseTopic, ok := rsp.Topics[topic]
+ if !ok {
+ errs = append(errs, ErrIncompleteResponse)
+ continue
+ }
+
+ for _, deleteRecordsResponsePartition := range deleteRecordsResponseTopic.Partitions {
+ if !errors.Is(deleteRecordsResponsePartition.Err, ErrNoError) {
+ errs = append(errs, deleteRecordsResponsePartition.Err)
+ continue
+ }
+ }
+ }
+ if len(errs) > 0 {
+ return Wrap(ErrDeleteRecords, errs...)
+ }
+ // todo since we are dealing with couple of partitions it would be good if we return slice of errors
+ // for each partition instead of one error
+ return nil
+}
+
+// Returns a bool indicating whether the resource request needs to go to a
+// specific broker
+func dependsOnSpecificNode(resource ConfigResource) bool {
+ return (resource.Type == BrokerResource && resource.Name != "") ||
+ resource.Type == BrokerLoggerResource
+}
+
+func (ca *clusterAdmin) DescribeConfig(resource ConfigResource) ([]ConfigEntry, error) {
+ var entries []ConfigEntry
+ var resources []*ConfigResource
+ resources = append(resources, &resource)
+
+ request := &DescribeConfigsRequest{
+ Resources: resources,
+ }
+
+ if ca.conf.Version.IsAtLeast(V1_1_0_0) {
+ request.Version = 1
+ }
+
+ if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ request.Version = 2
+ }
+
+ var (
+ b *Broker
+ err error
+ )
+
+ // DescribeConfig of broker/broker logger must be sent to the broker in question
+ if dependsOnSpecificNode(resource) {
+ var id int64
+ id, err = strconv.ParseInt(resource.Name, 10, 32)
+ if err != nil {
+ return nil, err
+ }
+ b, err = ca.findBroker(int32(id))
+ } else {
+ b, err = ca.findAnyBroker()
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ _ = b.Open(ca.client.Config())
+ rsp, err := b.DescribeConfigs(request)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, rspResource := range rsp.Resources {
+ if rspResource.Name == resource.Name {
+ if rspResource.ErrorCode != 0 {
+ return nil, &DescribeConfigError{Err: KError(rspResource.ErrorCode), ErrMsg: rspResource.ErrorMsg}
+ }
+ for _, cfgEntry := range rspResource.Configs {
+ entries = append(entries, *cfgEntry)
+ }
+ }
+ }
+ return entries, nil
+}
+
+func (ca *clusterAdmin) AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error {
+ var resources []*AlterConfigsResource
+ resources = append(resources, &AlterConfigsResource{
+ Type: resourceType,
+ Name: name,
+ ConfigEntries: entries,
+ })
+
+ request := &AlterConfigsRequest{
+ Resources: resources,
+ ValidateOnly: validateOnly,
+ }
+ if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ request.Version = 1
+ }
+
+ var (
+ b *Broker
+ err error
+ )
+
+ // AlterConfig of broker/broker logger must be sent to the broker in question
+ if dependsOnSpecificNode(ConfigResource{Name: name, Type: resourceType}) {
+ var id int64
+ id, err = strconv.ParseInt(name, 10, 32)
+ if err != nil {
+ return err
+ }
+ b, err = ca.findBroker(int32(id))
+ } else {
+ b, err = ca.findAnyBroker()
+ }
+ if err != nil {
+ return err
+ }
+
+ _ = b.Open(ca.client.Config())
+ rsp, err := b.AlterConfigs(request)
+ if err != nil {
+ return err
+ }
+
+ for _, rspResource := range rsp.Resources {
+ if rspResource.Name == name {
+ if rspResource.ErrorCode != 0 {
+ return &AlterConfigError{Err: KError(rspResource.ErrorCode), ErrMsg: rspResource.ErrorMsg}
+ }
+ }
+ }
+ return nil
+}
+
+func (ca *clusterAdmin) IncrementalAlterConfig(resourceType ConfigResourceType, name string, entries map[string]IncrementalAlterConfigsEntry, validateOnly bool) error {
+ var resources []*IncrementalAlterConfigsResource
+ resources = append(resources, &IncrementalAlterConfigsResource{
+ Type: resourceType,
+ Name: name,
+ ConfigEntries: entries,
+ })
+
+ request := &IncrementalAlterConfigsRequest{
+ Resources: resources,
+ ValidateOnly: validateOnly,
+ }
+
+ if ca.conf.Version.IsAtLeast(V2_4_0_0) {
+ request.Version = 1
+ }
+
+ var (
+ b *Broker
+ err error
+ )
+
+ // AlterConfig of broker/broker logger must be sent to the broker in question
+ if dependsOnSpecificNode(ConfigResource{Name: name, Type: resourceType}) {
+ var id int64
+ id, err = strconv.ParseInt(name, 10, 32)
+ if err != nil {
+ return err
+ }
+ b, err = ca.findBroker(int32(id))
+ } else {
+ b, err = ca.findAnyBroker()
+ }
+ if err != nil {
+ return err
+ }
+
+ _ = b.Open(ca.client.Config())
+ rsp, err := b.IncrementalAlterConfigs(request)
+ if err != nil {
+ return err
+ }
+
+ for _, rspResource := range rsp.Resources {
+ if rspResource.Name == name {
+ if rspResource.ErrorMsg != "" {
+ return errors.New(rspResource.ErrorMsg)
+ }
+ if rspResource.ErrorCode != 0 {
+ return KError(rspResource.ErrorCode)
+ }
+ }
+ }
+ return nil
+}
+
+func (ca *clusterAdmin) CreateACL(resource Resource, acl Acl) error {
+ var acls []*AclCreation
+ acls = append(acls, &AclCreation{resource, acl})
+ request := &CreateAclsRequest{AclCreations: acls}
+
+ if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ request.Version = 1
+ }
+
+ b, err := ca.Controller()
+ if err != nil {
+ return err
+ }
+
+ _, err = b.CreateAcls(request)
+ return err
+}
+
+func (ca *clusterAdmin) CreateACLs(resourceACLs []*ResourceAcls) error {
+ var acls []*AclCreation
+ for _, resourceACL := range resourceACLs {
+ for _, acl := range resourceACL.Acls {
+ acls = append(acls, &AclCreation{resourceACL.Resource, *acl})
+ }
+ }
+ request := &CreateAclsRequest{AclCreations: acls}
+
+ if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ request.Version = 1
+ }
+
+ b, err := ca.Controller()
+ if err != nil {
+ return err
+ }
+
+ _, err = b.CreateAcls(request)
+ return err
+}
+
+func (ca *clusterAdmin) ListAcls(filter AclFilter) ([]ResourceAcls, error) {
+ request := &DescribeAclsRequest{AclFilter: filter}
+
+ if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ request.Version = 1
+ }
+
+ b, err := ca.Controller()
+ if err != nil {
+ return nil, err
+ }
+
+ rsp, err := b.DescribeAcls(request)
+ if err != nil {
+ return nil, err
+ }
+
+ var lAcls []ResourceAcls
+ for _, rAcl := range rsp.ResourceAcls {
+ lAcls = append(lAcls, *rAcl)
+ }
+ return lAcls, nil
+}
+
+func (ca *clusterAdmin) DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error) {
+ var filters []*AclFilter
+ filters = append(filters, &filter)
+ request := &DeleteAclsRequest{Filters: filters}
+
+ if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ request.Version = 1
+ }
+
+ b, err := ca.Controller()
+ if err != nil {
+ return nil, err
+ }
+
+ rsp, err := b.DeleteAcls(request)
+ if err != nil {
+ return nil, err
+ }
+
+ var mAcls []MatchingAcl
+ for _, fr := range rsp.FilterResponses {
+ for _, mACL := range fr.MatchingAcls {
+ mAcls = append(mAcls, *mACL)
+ }
+ }
+ return mAcls, nil
+}
+
+func (ca *clusterAdmin) ElectLeaders(electionType ElectionType, partitions map[string][]int32) (map[string]map[int32]*PartitionResult, error) {
+ request := &ElectLeadersRequest{
+ Type: electionType,
+ TopicPartitions: partitions,
+ TimeoutMs: int32(60000),
+ }
+
+ if ca.conf.Version.IsAtLeast(V2_4_0_0) {
+ request.Version = 2
+ } else if ca.conf.Version.IsAtLeast(V0_11_0_0) {
+ request.Version = 1
+ }
+
+ var res *ElectLeadersResponse
+ if err := ca.retryOnError(isRetriableControllerError, func() error {
+ b, err := ca.Controller()
+ if err != nil {
+ return err
+ }
+ _ = b.Open(ca.client.Config())
+
+ res, err = b.ElectLeaders(request)
+ if err != nil {
+ return err
+ }
+ if !errors.Is(res.ErrorCode, ErrNoError) {
+ if isRetriableControllerError(res.ErrorCode) {
+ _, _ = ca.refreshController()
+ }
+ return res.ErrorCode
+ }
+ return nil
+ }); err != nil {
+ return nil, err
+ }
+ return res.ReplicaElectionResults, nil
+}
+
+func (ca *clusterAdmin) DescribeConsumerGroups(groups []string) (result []*GroupDescription, err error) {
+ groupsPerBroker := make(map[*Broker][]string)
+
+ for _, group := range groups {
+ coordinator, err := ca.client.Coordinator(group)
+ if err != nil {
+ return nil, err
+ }
+ groupsPerBroker[coordinator] = append(groupsPerBroker[coordinator], group)
+ }
+
+ for broker, brokerGroups := range groupsPerBroker {
+ describeReq := &DescribeGroupsRequest{
+ Groups: brokerGroups,
+ }
+
+ if ca.conf.Version.IsAtLeast(V2_4_0_0) {
+ // Starting in version 4, the response will include group.instance.id info for members.
+ // Starting in version 5, the response uses flexible encoding
+ describeReq.Version = 5
+ } else if ca.conf.Version.IsAtLeast(V2_3_0_0) {
+ // Starting in version 3, authorized operations can be requested.
+ describeReq.Version = 3
+ } else if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ // Version 2 is the same as version 0.
+ describeReq.Version = 2
+ } else if ca.conf.Version.IsAtLeast(V1_1_0_0) {
+ // Version 1 is the same as version 0.
+ describeReq.Version = 1
+ }
+ response, err := broker.DescribeGroups(describeReq)
+ if err != nil {
+ return nil, err
+ }
+
+ result = append(result, response.Groups...)
+ }
+ return result, nil
+}
+
+func (ca *clusterAdmin) ListConsumerGroups() (allGroups map[string]string, err error) {
+ allGroups = make(map[string]string)
+
+ // Query brokers in parallel, since we have to query *all* brokers
+ brokers := ca.client.Brokers()
+ groupMaps := make(chan map[string]string, len(brokers))
+ errChan := make(chan error, len(brokers))
+ wg := sync.WaitGroup{}
+
+ for _, b := range brokers {
+ wg.Add(1)
+ go func(b *Broker, conf *Config) {
+ defer wg.Done()
+ _ = b.Open(conf) // Ensure that broker is opened
+
+ request := &ListGroupsRequest{}
+ if ca.conf.Version.IsAtLeast(V3_8_0_0) {
+ // Version 5 adds the TypesFilter field (KIP-848).
+ request.Version = 5
+ } else if ca.conf.Version.IsAtLeast(V2_6_0_0) {
+ // Version 4 adds the StatesFilter field (KIP-518).
+ request.Version = 4
+ } else if ca.conf.Version.IsAtLeast(V2_4_0_0) {
+ // Version 3 is the first flexible version.
+ request.Version = 3
+ } else if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ // Version 2 is the same as version 0.
+ request.Version = 2
+ } else if ca.conf.Version.IsAtLeast(V0_11_0_0) {
+ // Version 1 is the same as version 0.
+ request.Version = 1
+ }
+
+ response, err := b.ListGroups(request)
+ if err != nil {
+ errChan <- err
+ return
+ }
+
+ groupMaps <- maps.Clone(response.Groups)
+ }(b, ca.conf)
+ }
+
+ wg.Wait()
+ close(groupMaps)
+ close(errChan)
+
+ for groupMap := range groupMaps {
+ maps.Copy(allGroups, groupMap)
+ }
+
+ // Intentionally return only the first error for simplicity
+ err = <-errChan
+ return
+}
+
+func (ca *clusterAdmin) ListConsumerGroupOffsets(group string, topicPartitions map[string][]int32) (*OffsetFetchResponse, error) {
+ var response *OffsetFetchResponse
+ request := NewOffsetFetchRequest(ca.conf.Version, group, topicPartitions)
+ err := ca.retryOnError(isRetriableGroupCoordinatorError, func() (err error) {
+ defer func() {
+ if err != nil && isRetriableGroupCoordinatorError(err) {
+ _ = ca.client.RefreshCoordinator(group)
+ }
+ }()
+
+ coordinator, err := ca.client.Coordinator(group)
+ if err != nil {
+ return err
+ }
+
+ response, err = coordinator.FetchOffset(request)
+ if err != nil {
+ return err
+ }
+ if !errors.Is(response.Err, ErrNoError) {
+ return response.Err
+ }
+
+ return nil
+ })
+
+ return response, err
+}
+
+func (ca *clusterAdmin) DeleteConsumerGroupOffset(group string, topic string, partition int32) error {
+ var response *DeleteOffsetsResponse
+ request := &DeleteOffsetsRequest{
+ Group: group,
+ partitions: map[string][]int32{
+ topic: {partition},
+ },
+ }
+
+ return ca.retryOnError(isRetriableGroupCoordinatorError, func() (err error) {
+ defer func() {
+ if err != nil && isRetriableGroupCoordinatorError(err) {
+ _ = ca.client.RefreshCoordinator(group)
+ }
+ }()
+
+ coordinator, err := ca.client.Coordinator(group)
+ if err != nil {
+ return err
+ }
+
+ response, err = coordinator.DeleteOffsets(request)
+ if err != nil {
+ return err
+ }
+ if !errors.Is(response.ErrorCode, ErrNoError) {
+ return response.ErrorCode
+ }
+ if !errors.Is(response.Errors[topic][partition], ErrNoError) {
+ return response.Errors[topic][partition]
+ }
+
+ return nil
+ })
+}
+
+func (ca *clusterAdmin) DeleteConsumerGroup(group string) error {
+ var response *DeleteGroupsResponse
+ request := &DeleteGroupsRequest{
+ Groups: []string{group},
+ }
+
+ if ca.conf.Version.IsAtLeast(V2_4_0_0) {
+ request.Version = 2
+ } else if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ request.Version = 1
+ }
+
+ return ca.retryOnError(isRetriableGroupCoordinatorError, func() (err error) {
+ defer func() {
+ if err != nil && isRetriableGroupCoordinatorError(err) {
+ _ = ca.client.RefreshCoordinator(group)
+ }
+ }()
+
+ coordinator, err := ca.client.Coordinator(group)
+ if err != nil {
+ return err
+ }
+
+ response, err = coordinator.DeleteGroups(request)
+ if err != nil {
+ return err
+ }
+
+ groupErr, ok := response.GroupErrorCodes[group]
+ if !ok {
+ return ErrIncompleteResponse
+ }
+
+ if !errors.Is(groupErr, ErrNoError) {
+ return groupErr
+ }
+
+ return nil
+ })
+}
+
+func (ca *clusterAdmin) DescribeLogDirs(brokerIds []int32) (allLogDirs map[int32][]DescribeLogDirsResponseDirMetadata, err error) {
+ type result struct {
+ id int32
+ logdirs []DescribeLogDirsResponseDirMetadata
+ }
+ // Query brokers in parallel, since we may have to query multiple brokers
+ logDirsResults := make(chan result, len(brokerIds))
+ errChan := make(chan error, len(brokerIds))
+ wg := sync.WaitGroup{}
+
+ for _, b := range brokerIds {
+ broker, err := ca.findBroker(b)
+ if err != nil {
+ Logger.Printf("Unable to find broker with ID = %v\n", b)
+ continue
+ }
+ wg.Add(1)
+ go func(b *Broker, conf *Config) {
+ defer wg.Done()
+ _ = b.Open(conf) // Ensure that broker is opened
+
+ request := &DescribeLogDirsRequest{}
+ if ca.conf.Version.IsAtLeast(V3_3_0_0) {
+ request.Version = 4
+ } else if ca.conf.Version.IsAtLeast(V3_2_0_0) {
+ request.Version = 3
+ } else if ca.conf.Version.IsAtLeast(V2_6_0_0) {
+ request.Version = 2
+ } else if ca.conf.Version.IsAtLeast(V2_0_0_0) {
+ request.Version = 1
+ }
+ response, err := b.DescribeLogDirs(request)
+ if err != nil {
+ errChan <- err
+ return
+ }
+ if !errors.Is(response.ErrorCode, ErrNoError) {
+ errChan <- response.ErrorCode
+ return
+ }
+ logDirsResults <- result{id: b.ID(), logdirs: response.LogDirs}
+ }(broker, ca.conf)
+ }
+
+ wg.Wait()
+ close(logDirsResults)
+ close(errChan)
+
+ allLogDirs = make(map[int32][]DescribeLogDirsResponseDirMetadata, len(brokerIds))
+ for logDirsResult := range logDirsResults {
+ allLogDirs[logDirsResult.id] = logDirsResult.logdirs
+ }
+
+ // Intentionally return only the first error for simplicity
+ err = <-errChan
+ return
+}
+
+func (ca *clusterAdmin) DescribeUserScramCredentials(users []string) ([]*DescribeUserScramCredentialsResult, error) {
+ req := &DescribeUserScramCredentialsRequest{}
+ for _, u := range users {
+ req.DescribeUsers = append(req.DescribeUsers, DescribeUserScramCredentialsRequestUser{
+ Name: u,
+ })
+ }
+
+ b, err := ca.Controller()
+ if err != nil {
+ return nil, err
+ }
+
+ rsp, err := b.DescribeUserScramCredentials(req)
+ if err != nil {
+ return nil, err
+ }
+
+ return rsp.Results, nil
+}
+
+func (ca *clusterAdmin) UpsertUserScramCredentials(upsert []AlterUserScramCredentialsUpsert) ([]*AlterUserScramCredentialsResult, error) {
+ res, err := ca.AlterUserScramCredentials(upsert, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ return res, nil
+}
+
+func (ca *clusterAdmin) DeleteUserScramCredentials(delete []AlterUserScramCredentialsDelete) ([]*AlterUserScramCredentialsResult, error) {
+ res, err := ca.AlterUserScramCredentials(nil, delete)
+ if err != nil {
+ return nil, err
+ }
+
+ return res, nil
+}
+
+func (ca *clusterAdmin) AlterUserScramCredentials(u []AlterUserScramCredentialsUpsert, d []AlterUserScramCredentialsDelete) ([]*AlterUserScramCredentialsResult, error) {
+ req := &AlterUserScramCredentialsRequest{
+ Deletions: d,
+ Upsertions: u,
+ }
+
+ var rsp *AlterUserScramCredentialsResponse
+ err := ca.retryOnError(isRetriableControllerError, func() error {
+ b, err := ca.Controller()
+ if err != nil {
+ return err
+ }
+
+ rsp, err = b.AlterUserScramCredentials(req)
+ return err
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ return rsp.Results, nil
+}
+
+// Describe All : use an empty/nil components slice + strict = false
+// Contains components: strict = false
+// Contains only components: strict = true
+func (ca *clusterAdmin) DescribeClientQuotas(components []QuotaFilterComponent, strict bool) ([]DescribeClientQuotasEntry, error) {
+ request := NewDescribeClientQuotasRequest(
+ ca.conf.Version,
+ components,
+ strict,
+ )
+
+ b, err := ca.Controller()
+ if err != nil {
+ return nil, err
+ }
+
+ rsp, err := b.DescribeClientQuotas(request)
+ if err != nil {
+ return nil, err
+ }
+
+ if rsp.ErrorMsg != nil && len(*rsp.ErrorMsg) > 0 {
+ return nil, errors.New(*rsp.ErrorMsg)
+ }
+ if !errors.Is(rsp.ErrorCode, ErrNoError) {
+ return nil, rsp.ErrorCode
+ }
+
+ return rsp.Entries, nil
+}
+
+func (ca *clusterAdmin) AlterClientQuotas(entity []QuotaEntityComponent, op ClientQuotasOp, validateOnly bool) error {
+ entry := AlterClientQuotasEntry{
+ Entity: entity,
+ Ops: []ClientQuotasOp{op},
+ }
+
+ request := &AlterClientQuotasRequest{
+ Entries: []AlterClientQuotasEntry{entry},
+ ValidateOnly: validateOnly,
+ }
+
+ b, err := ca.Controller()
+ if err != nil {
+ return err
+ }
+
+ rsp, err := b.AlterClientQuotas(request)
+ if err != nil {
+ return err
+ }
+
+ for _, entry := range rsp.Entries {
+ if entry.ErrorMsg != nil && len(*entry.ErrorMsg) > 0 {
+ return errors.New(*entry.ErrorMsg)
+ }
+ if !errors.Is(entry.ErrorCode, ErrNoError) {
+ return entry.ErrorCode
+ }
+ }
+
+ return nil
+}
+
+func (ca *clusterAdmin) RemoveMemberFromConsumerGroup(group string, groupInstanceIds []string) (*LeaveGroupResponse, error) {
+ if !ca.conf.Version.IsAtLeast(V2_4_0_0) {
+ return nil, ConfigurationError("Removing members from a consumer group headers requires Kafka version of at least v2.4.0")
+ }
+ var response *LeaveGroupResponse
+ request := &LeaveGroupRequest{
+ Version: 3,
+ GroupId: group,
+ }
+ for _, instanceId := range groupInstanceIds {
+ groupInstanceId := instanceId
+ request.Members = append(request.Members, MemberIdentity{
+ GroupInstanceId: &groupInstanceId,
+ })
+ }
+ err := ca.retryOnError(isRetriableGroupCoordinatorError, func() (err error) {
+ defer func() {
+ if err != nil && isRetriableGroupCoordinatorError(err) {
+ _ = ca.client.RefreshCoordinator(group)
+ }
+ }()
+
+ coordinator, err := ca.client.Coordinator(group)
+ if err != nil {
+ return err
+ }
+
+ response, err = coordinator.LeaveGroup(request)
+ if err != nil {
+ return err
+ }
+ if !errors.Is(response.Err, ErrNoError) {
+ return response.Err
+ }
+
+ return nil
+ })
+
+ return response, err
+}
diff --git a/vendor/github.com/IBM/sarama/alter_client_quotas_request.go b/vendor/github.com/IBM/sarama/alter_client_quotas_request.go
new file mode 100644
index 0000000..95554e3
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/alter_client_quotas_request.go
@@ -0,0 +1,203 @@
+package sarama
+
+// AlterClientQuotas Request (Version: 0) => [entries] validate_only
+// entries => [entity] [ops]
+// entity => entity_type entity_name
+// entity_type => STRING
+// entity_name => NULLABLE_STRING
+// ops => key value remove
+// key => STRING
+// value => FLOAT64
+// remove => BOOLEAN
+// validate_only => BOOLEAN
+
+type AlterClientQuotasRequest struct {
+ Version int16
+ Entries []AlterClientQuotasEntry // The quota configuration entries to alter.
+ ValidateOnly bool // Whether the alteration should be validated, but not performed.
+}
+
+func (a *AlterClientQuotasRequest) setVersion(v int16) {
+ a.Version = v
+}
+
+type AlterClientQuotasEntry struct {
+ Entity []QuotaEntityComponent // The quota entity to alter.
+ Ops []ClientQuotasOp // An individual quota configuration entry to alter.
+}
+
+type ClientQuotasOp struct {
+ Key string // The quota configuration key.
+ Value float64 // The value to set, otherwise ignored if the value is to be removed.
+ Remove bool // Whether the quota configuration value should be removed, otherwise set.
+}
+
+func (a *AlterClientQuotasRequest) encode(pe packetEncoder) error {
+ // Entries
+ if err := pe.putArrayLength(len(a.Entries)); err != nil {
+ return err
+ }
+ for _, e := range a.Entries {
+ if err := e.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ // ValidateOnly
+ pe.putBool(a.ValidateOnly)
+
+ return nil
+}
+
+func (a *AlterClientQuotasRequest) decode(pd packetDecoder, version int16) error {
+ // Entries
+ entryCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if entryCount > 0 {
+ a.Entries = make([]AlterClientQuotasEntry, entryCount)
+ for i := range a.Entries {
+ e := AlterClientQuotasEntry{}
+ if err = e.decode(pd, version); err != nil {
+ return err
+ }
+ a.Entries[i] = e
+ }
+ } else {
+ a.Entries = []AlterClientQuotasEntry{}
+ }
+
+ // ValidateOnly
+ validateOnly, err := pd.getBool()
+ if err != nil {
+ return err
+ }
+ a.ValidateOnly = validateOnly
+
+ return nil
+}
+
+func (a *AlterClientQuotasEntry) encode(pe packetEncoder) error {
+ // Entity
+ if err := pe.putArrayLength(len(a.Entity)); err != nil {
+ return err
+ }
+ for _, component := range a.Entity {
+ if err := component.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ // Ops
+ if err := pe.putArrayLength(len(a.Ops)); err != nil {
+ return err
+ }
+ for _, o := range a.Ops {
+ if err := o.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (a *AlterClientQuotasEntry) decode(pd packetDecoder, version int16) error {
+ // Entity
+ componentCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if componentCount > 0 {
+ a.Entity = make([]QuotaEntityComponent, componentCount)
+ for i := 0; i < componentCount; i++ {
+ component := QuotaEntityComponent{}
+ if err := component.decode(pd, version); err != nil {
+ return err
+ }
+ a.Entity[i] = component
+ }
+ } else {
+ a.Entity = []QuotaEntityComponent{}
+ }
+
+ // Ops
+ opCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if opCount > 0 {
+ a.Ops = make([]ClientQuotasOp, opCount)
+ for i := range a.Ops {
+ c := ClientQuotasOp{}
+ if err = c.decode(pd, version); err != nil {
+ return err
+ }
+ a.Ops[i] = c
+ }
+ } else {
+ a.Ops = []ClientQuotasOp{}
+ }
+
+ return nil
+}
+
+func (c *ClientQuotasOp) encode(pe packetEncoder) error {
+ // Key
+ if err := pe.putString(c.Key); err != nil {
+ return err
+ }
+
+ // Value
+ pe.putFloat64(c.Value)
+
+ // Remove
+ pe.putBool(c.Remove)
+
+ return nil
+}
+
+func (c *ClientQuotasOp) decode(pd packetDecoder, version int16) error {
+ // Key
+ key, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ c.Key = key
+
+ // Value
+ value, err := pd.getFloat64()
+ if err != nil {
+ return err
+ }
+ c.Value = value
+
+ // Remove
+ remove, err := pd.getBool()
+ if err != nil {
+ return err
+ }
+ c.Remove = remove
+
+ return nil
+}
+
+func (a *AlterClientQuotasRequest) key() int16 {
+ return apiKeyAlterClientQuotas
+}
+
+func (a *AlterClientQuotasRequest) version() int16 {
+ return a.Version
+}
+
+func (a *AlterClientQuotasRequest) headerVersion() int16 {
+ return 1
+}
+
+func (a *AlterClientQuotasRequest) isValidVersion() bool {
+ return a.Version == 0
+}
+
+func (a *AlterClientQuotasRequest) requiredVersion() KafkaVersion {
+ return V2_6_0_0
+}
diff --git a/vendor/github.com/IBM/sarama/alter_client_quotas_response.go b/vendor/github.com/IBM/sarama/alter_client_quotas_response.go
new file mode 100644
index 0000000..2ca4974
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/alter_client_quotas_response.go
@@ -0,0 +1,153 @@
+package sarama
+
+import (
+ "time"
+)
+
+// AlterClientQuotas Response (Version: 0) => throttle_time_ms [entries]
+// throttle_time_ms => INT32
+// entries => error_code error_message [entity]
+// error_code => INT16
+// error_message => NULLABLE_STRING
+// entity => entity_type entity_name
+// entity_type => STRING
+// entity_name => NULLABLE_STRING
+
+type AlterClientQuotasResponse struct {
+ Version int16
+ ThrottleTime time.Duration // The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota.
+ Entries []AlterClientQuotasEntryResponse // The quota configuration entries altered.
+}
+
+func (a *AlterClientQuotasResponse) setVersion(v int16) {
+ a.Version = v
+}
+
+type AlterClientQuotasEntryResponse struct {
+ ErrorCode KError // The error code, or `0` if the quota alteration succeeded.
+ ErrorMsg *string // The error message, or `null` if the quota alteration succeeded.
+ Entity []QuotaEntityComponent // The quota entity altered.
+}
+
+func (a *AlterClientQuotasResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(a.ThrottleTime)
+
+ // Entries
+ if err := pe.putArrayLength(len(a.Entries)); err != nil {
+ return err
+ }
+ for _, e := range a.Entries {
+ if err := e.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (a *AlterClientQuotasResponse) decode(pd packetDecoder, version int16) (err error) {
+ if a.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ // Entries
+ entryCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if entryCount > 0 {
+ a.Entries = make([]AlterClientQuotasEntryResponse, entryCount)
+ for i := range a.Entries {
+ e := AlterClientQuotasEntryResponse{}
+ if err = e.decode(pd, version); err != nil {
+ return err
+ }
+ a.Entries[i] = e
+ }
+ } else {
+ a.Entries = []AlterClientQuotasEntryResponse{}
+ }
+
+ return nil
+}
+
+func (a *AlterClientQuotasEntryResponse) encode(pe packetEncoder) error {
+ // ErrorCode
+ pe.putKError(a.ErrorCode)
+
+ // ErrorMsg
+ if err := pe.putNullableString(a.ErrorMsg); err != nil {
+ return err
+ }
+
+ // Entity
+ if err := pe.putArrayLength(len(a.Entity)); err != nil {
+ return err
+ }
+ for _, component := range a.Entity {
+ if err := component.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (a *AlterClientQuotasEntryResponse) decode(pd packetDecoder, version int16) (err error) {
+ // ErrorCode
+ a.ErrorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ // ErrorMsg
+ errMsg, err := pd.getNullableString()
+ if err != nil {
+ return err
+ }
+ a.ErrorMsg = errMsg
+
+ // Entity
+ componentCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if componentCount > 0 {
+ a.Entity = make([]QuotaEntityComponent, componentCount)
+ for i := 0; i < componentCount; i++ {
+ component := QuotaEntityComponent{}
+ if err := component.decode(pd, version); err != nil {
+ return err
+ }
+ a.Entity[i] = component
+ }
+ } else {
+ a.Entity = []QuotaEntityComponent{}
+ }
+
+ return nil
+}
+
+func (a *AlterClientQuotasResponse) key() int16 {
+ return apiKeyAlterClientQuotas
+}
+
+func (a *AlterClientQuotasResponse) version() int16 {
+ return a.Version
+}
+
+func (a *AlterClientQuotasResponse) headerVersion() int16 {
+ return 0
+}
+
+func (a *AlterClientQuotasResponse) isValidVersion() bool {
+ return a.Version == 0
+}
+
+func (a *AlterClientQuotasResponse) requiredVersion() KafkaVersion {
+ return V2_6_0_0
+}
+
+func (r *AlterClientQuotasResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/alter_configs_request.go b/vendor/github.com/IBM/sarama/alter_configs_request.go
new file mode 100644
index 0000000..5293b49
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/alter_configs_request.go
@@ -0,0 +1,142 @@
+package sarama
+
+// AlterConfigsRequest is an alter config request type
+type AlterConfigsRequest struct {
+ Version int16
+ Resources []*AlterConfigsResource
+ ValidateOnly bool
+}
+
+func (a *AlterConfigsRequest) setVersion(v int16) {
+ a.Version = v
+}
+
+// AlterConfigsResource is an alter config resource type
+type AlterConfigsResource struct {
+ Type ConfigResourceType
+ Name string
+ ConfigEntries map[string]*string
+}
+
+func (a *AlterConfigsRequest) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(a.Resources)); err != nil {
+ return err
+ }
+
+ for _, r := range a.Resources {
+ if err := r.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ pe.putBool(a.ValidateOnly)
+ return nil
+}
+
+func (a *AlterConfigsRequest) decode(pd packetDecoder, version int16) error {
+ resourceCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ a.Resources = make([]*AlterConfigsResource, resourceCount)
+ for i := range a.Resources {
+ r := &AlterConfigsResource{}
+ err = r.decode(pd, version)
+ if err != nil {
+ return err
+ }
+ a.Resources[i] = r
+ }
+
+ validateOnly, err := pd.getBool()
+ if err != nil {
+ return err
+ }
+
+ a.ValidateOnly = validateOnly
+
+ return nil
+}
+
+func (a *AlterConfigsResource) encode(pe packetEncoder) error {
+ pe.putInt8(int8(a.Type))
+
+ if err := pe.putString(a.Name); err != nil {
+ return err
+ }
+
+ if err := pe.putArrayLength(len(a.ConfigEntries)); err != nil {
+ return err
+ }
+ for configKey, configValue := range a.ConfigEntries {
+ if err := pe.putString(configKey); err != nil {
+ return err
+ }
+ if err := pe.putNullableString(configValue); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (a *AlterConfigsResource) decode(pd packetDecoder, version int16) error {
+ t, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ a.Type = ConfigResourceType(t)
+
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ a.Name = name
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if n > 0 {
+ a.ConfigEntries = make(map[string]*string, n)
+ for i := 0; i < n; i++ {
+ configKey, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ if a.ConfigEntries[configKey], err = pd.getNullableString(); err != nil {
+ return err
+ }
+ }
+ }
+ return err
+}
+
+func (a *AlterConfigsRequest) key() int16 {
+ return apiKeyAlterConfigs
+}
+
+func (a *AlterConfigsRequest) version() int16 {
+ return a.Version
+}
+
+func (a *AlterConfigsRequest) headerVersion() int16 {
+ return 1
+}
+
+func (a *AlterConfigsRequest) isValidVersion() bool {
+ return a.Version >= 0 && a.Version <= 1
+}
+
+func (a *AlterConfigsRequest) requiredVersion() KafkaVersion {
+ switch a.Version {
+ case 1:
+ return V2_0_0_0
+ case 0:
+ return V0_11_0_0
+ default:
+ return V2_0_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/alter_configs_response.go b/vendor/github.com/IBM/sarama/alter_configs_response.go
new file mode 100644
index 0000000..cde2686
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/alter_configs_response.go
@@ -0,0 +1,156 @@
+package sarama
+
+import (
+ "fmt"
+ "time"
+)
+
+// AlterConfigsResponse is a response type for alter config
+type AlterConfigsResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ Resources []*AlterConfigsResourceResponse
+}
+
+func (a *AlterConfigsResponse) setVersion(v int16) {
+ a.Version = v
+}
+
+type AlterConfigError struct {
+ Err KError
+ ErrMsg string
+}
+
+func (c *AlterConfigError) Error() string {
+ text := c.Err.Error()
+ if c.ErrMsg != "" {
+ text = fmt.Sprintf("%s - %s", text, c.ErrMsg)
+ }
+ return text
+}
+
+// AlterConfigsResourceResponse is a response type for alter config resource
+type AlterConfigsResourceResponse struct {
+ ErrorCode int16
+ ErrorMsg string
+ Type ConfigResourceType
+ Name string
+}
+
+func (a *AlterConfigsResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(a.ThrottleTime)
+
+ if err := pe.putArrayLength(len(a.Resources)); err != nil {
+ return err
+ }
+
+ for _, v := range a.Resources {
+ if err := v.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (a *AlterConfigsResponse) decode(pd packetDecoder, version int16) (err error) {
+ if a.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ responseCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ a.Resources = make([]*AlterConfigsResourceResponse, responseCount)
+
+ for i := range a.Resources {
+ a.Resources[i] = new(AlterConfigsResourceResponse)
+
+ if err := a.Resources[i].decode(pd, version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (a *AlterConfigsResourceResponse) encode(pe packetEncoder) error {
+ pe.putInt16(a.ErrorCode)
+ err := pe.putString(a.ErrorMsg)
+ if err != nil {
+ return err
+ }
+ pe.putInt8(int8(a.Type))
+ err = pe.putString(a.Name)
+ if err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (a *AlterConfigsResourceResponse) decode(pd packetDecoder, version int16) error {
+ errCode, err := pd.getInt16()
+ if err != nil {
+ return err
+ }
+ a.ErrorCode = errCode
+
+ e, err := pd.getNullableString()
+ if err != nil {
+ return err
+ }
+ if e == nil {
+ a.ErrorMsg = ""
+ } else {
+ a.ErrorMsg = *e
+ }
+
+ t, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ a.Type = ConfigResourceType(t)
+
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ a.Name = name
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (a *AlterConfigsResponse) key() int16 {
+ return apiKeyAlterConfigs
+}
+
+func (a *AlterConfigsResponse) version() int16 {
+ return a.Version
+}
+
+func (a *AlterConfigsResponse) headerVersion() int16 {
+ return 0
+}
+
+func (a *AlterConfigsResponse) isValidVersion() bool {
+ return a.Version >= 0 && a.Version <= 1
+}
+
+func (a *AlterConfigsResponse) requiredVersion() KafkaVersion {
+ switch a.Version {
+ case 1:
+ return V2_0_0_0
+ case 0:
+ return V0_11_0_0
+ default:
+ return V2_0_0_0
+ }
+}
+
+func (r *AlterConfigsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/alter_partition_reassignments_request.go b/vendor/github.com/IBM/sarama/alter_partition_reassignments_request.go
new file mode 100644
index 0000000..165cbcc
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/alter_partition_reassignments_request.go
@@ -0,0 +1,146 @@
+package sarama
+
+type alterPartitionReassignmentsBlock struct {
+ replicas []int32
+}
+
+func (b *alterPartitionReassignmentsBlock) encode(pe packetEncoder) error {
+ if err := pe.putNullableInt32Array(b.replicas); err != nil {
+ return err
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (b *alterPartitionReassignmentsBlock) decode(pd packetDecoder) (err error) {
+ if b.replicas, err = pd.getInt32Array(); err != nil {
+ return err
+ }
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ return nil
+}
+
+type AlterPartitionReassignmentsRequest struct {
+ TimeoutMs int32
+ blocks map[string]map[int32]*alterPartitionReassignmentsBlock
+ Version int16
+}
+
+func (r *AlterPartitionReassignmentsRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *AlterPartitionReassignmentsRequest) encode(pe packetEncoder) error {
+ pe.putInt32(r.TimeoutMs)
+
+ if err := pe.putArrayLength(len(r.blocks)); err != nil {
+ return err
+ }
+
+ for topic, partitions := range r.blocks {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putArrayLength(len(partitions)); err != nil {
+ return err
+ }
+ for partition, block := range partitions {
+ pe.putInt32(partition)
+ if err := block.encode(pe); err != nil {
+ return err
+ }
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+
+ return nil
+}
+
+func (r *AlterPartitionReassignmentsRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+
+ if r.TimeoutMs, err = pd.getInt32(); err != nil {
+ return err
+ }
+
+ topicCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if topicCount > 0 {
+ r.blocks = make(map[string]map[int32]*alterPartitionReassignmentsBlock)
+ for i := 0; i < topicCount; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ partitionCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ r.blocks[topic] = make(map[int32]*alterPartitionReassignmentsBlock)
+ for j := 0; j < partitionCount; j++ {
+ partition, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ block := &alterPartitionReassignmentsBlock{}
+ if err := block.decode(pd); err != nil {
+ return err
+ }
+ r.blocks[topic][partition] = block
+ }
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *AlterPartitionReassignmentsRequest) key() int16 {
+ return apiKeyAlterPartitionReassignments
+}
+
+func (r *AlterPartitionReassignmentsRequest) version() int16 {
+ return r.Version
+}
+
+func (r *AlterPartitionReassignmentsRequest) headerVersion() int16 {
+ return 2
+}
+
+func (r *AlterPartitionReassignmentsRequest) isValidVersion() bool {
+ return r.Version == 0
+}
+
+func (r *AlterPartitionReassignmentsRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *AlterPartitionReassignmentsRequest) isFlexibleVersion(version int16) bool {
+ return version >= 0
+}
+
+func (r *AlterPartitionReassignmentsRequest) requiredVersion() KafkaVersion {
+ return V2_4_0_0
+}
+
+func (r *AlterPartitionReassignmentsRequest) AddBlock(topic string, partitionID int32, replicas []int32) {
+ if r.blocks == nil {
+ r.blocks = make(map[string]map[int32]*alterPartitionReassignmentsBlock)
+ }
+
+ if r.blocks[topic] == nil {
+ r.blocks[topic] = make(map[int32]*alterPartitionReassignmentsBlock)
+ }
+
+ r.blocks[topic][partitionID] = &alterPartitionReassignmentsBlock{replicas}
+}
diff --git a/vendor/github.com/IBM/sarama/alter_partition_reassignments_response.go b/vendor/github.com/IBM/sarama/alter_partition_reassignments_response.go
new file mode 100644
index 0000000..886c4f6
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/alter_partition_reassignments_response.go
@@ -0,0 +1,178 @@
+package sarama
+
+import "time"
+
+type alterPartitionReassignmentsErrorBlock struct {
+ errorCode KError
+ errorMessage *string
+}
+
+func (b *alterPartitionReassignmentsErrorBlock) encode(pe packetEncoder) error {
+ pe.putKError(b.errorCode)
+ if err := pe.putNullableString(b.errorMessage); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+
+ return nil
+}
+
+func (b *alterPartitionReassignmentsErrorBlock) decode(pd packetDecoder) (err error) {
+ b.errorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+ b.errorMessage, err = pd.getNullableString()
+ if err != nil {
+ return err
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+type AlterPartitionReassignmentsResponse struct {
+ Version int16
+ ThrottleTimeMs int32
+ ErrorCode KError
+ ErrorMessage *string
+ Errors map[string]map[int32]*alterPartitionReassignmentsErrorBlock
+}
+
+func (r *AlterPartitionReassignmentsResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *AlterPartitionReassignmentsResponse) AddError(topic string, partition int32, kerror KError, message *string) {
+ if r.Errors == nil {
+ r.Errors = make(map[string]map[int32]*alterPartitionReassignmentsErrorBlock)
+ }
+ partitions := r.Errors[topic]
+ if partitions == nil {
+ partitions = make(map[int32]*alterPartitionReassignmentsErrorBlock)
+ r.Errors[topic] = partitions
+ }
+
+ partitions[partition] = &alterPartitionReassignmentsErrorBlock{errorCode: kerror, errorMessage: message}
+}
+
+func (r *AlterPartitionReassignmentsResponse) encode(pe packetEncoder) error {
+ pe.putInt32(r.ThrottleTimeMs)
+ pe.putKError(r.ErrorCode)
+ if err := pe.putNullableString(r.ErrorMessage); err != nil {
+ return err
+ }
+
+ if err := pe.putArrayLength(len(r.Errors)); err != nil {
+ return err
+ }
+ for topic, partitions := range r.Errors {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putArrayLength(len(partitions)); err != nil {
+ return err
+ }
+ for partition, block := range partitions {
+ pe.putInt32(partition)
+
+ if err := block.encode(pe); err != nil {
+ return err
+ }
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *AlterPartitionReassignmentsResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+
+ if r.ThrottleTimeMs, err = pd.getInt32(); err != nil {
+ return err
+ }
+
+ r.ErrorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if r.ErrorMessage, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ numTopics, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if numTopics > 0 {
+ r.Errors = make(map[string]map[int32]*alterPartitionReassignmentsErrorBlock, numTopics)
+ for i := 0; i < numTopics; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ ongoingPartitionReassignments, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Errors[topic] = make(map[int32]*alterPartitionReassignmentsErrorBlock, ongoingPartitionReassignments)
+
+ for j := 0; j < ongoingPartitionReassignments; j++ {
+ partition, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ block := &alterPartitionReassignmentsErrorBlock{}
+ if err := block.decode(pd); err != nil {
+ return err
+ }
+
+ r.Errors[topic][partition] = block
+ }
+ if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *AlterPartitionReassignmentsResponse) key() int16 {
+ return apiKeyAlterPartitionReassignments
+}
+
+func (r *AlterPartitionReassignmentsResponse) version() int16 {
+ return r.Version
+}
+
+func (r *AlterPartitionReassignmentsResponse) headerVersion() int16 {
+ return 1
+}
+
+func (r *AlterPartitionReassignmentsResponse) isValidVersion() bool {
+ return r.Version == 0
+}
+
+func (r *AlterPartitionReassignmentsResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *AlterPartitionReassignmentsResponse) isFlexibleVersion(version int16) bool {
+ return version >= 0
+}
+
+func (r *AlterPartitionReassignmentsResponse) requiredVersion() KafkaVersion {
+ return V2_4_0_0
+}
+
+func (r *AlterPartitionReassignmentsResponse) throttleTime() time.Duration {
+ return time.Duration(r.ThrottleTimeMs) * time.Millisecond
+}
diff --git a/vendor/github.com/IBM/sarama/alter_user_scram_credentials_request.go b/vendor/github.com/IBM/sarama/alter_user_scram_credentials_request.go
new file mode 100644
index 0000000..d50a38d
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/alter_user_scram_credentials_request.go
@@ -0,0 +1,160 @@
+package sarama
+
+type AlterUserScramCredentialsRequest struct {
+ Version int16
+
+ // Deletions represent list of SCRAM credentials to remove
+ Deletions []AlterUserScramCredentialsDelete
+
+ // Upsertions represent list of SCRAM credentials to update/insert
+ Upsertions []AlterUserScramCredentialsUpsert
+}
+
+func (r *AlterUserScramCredentialsRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+type AlterUserScramCredentialsDelete struct {
+ Name string
+ Mechanism ScramMechanismType
+}
+
+type AlterUserScramCredentialsUpsert struct {
+ Name string
+ Mechanism ScramMechanismType
+ Iterations int32
+ Salt []byte
+ saltedPassword []byte
+
+ // This field is never transmitted over the wire
+ // @see: https://tools.ietf.org/html/rfc5802
+ Password []byte
+}
+
+func (r *AlterUserScramCredentialsRequest) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(r.Deletions)); err != nil {
+ return err
+ }
+ for _, d := range r.Deletions {
+ if err := pe.putString(d.Name); err != nil {
+ return err
+ }
+ pe.putInt8(int8(d.Mechanism))
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ if err := pe.putArrayLength(len(r.Upsertions)); err != nil {
+ return err
+ }
+ for _, u := range r.Upsertions {
+ if err := pe.putString(u.Name); err != nil {
+ return err
+ }
+ pe.putInt8(int8(u.Mechanism))
+ pe.putInt32(u.Iterations)
+
+ if err := pe.putBytes(u.Salt); err != nil {
+ return err
+ }
+
+ // do not transmit the password over the wire
+ formatter := scramFormatter{mechanism: u.Mechanism}
+ salted, err := formatter.saltedPassword(u.Password, u.Salt, int(u.Iterations))
+ if err != nil {
+ return err
+ }
+
+ if err := pe.putBytes(salted); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *AlterUserScramCredentialsRequest) decode(pd packetDecoder, version int16) error {
+ numDeletions, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Deletions = make([]AlterUserScramCredentialsDelete, numDeletions)
+ for i := 0; i < numDeletions; i++ {
+ r.Deletions[i] = AlterUserScramCredentialsDelete{}
+ if r.Deletions[i].Name, err = pd.getString(); err != nil {
+ return err
+ }
+ mechanism, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ r.Deletions[i].Mechanism = ScramMechanismType(mechanism)
+ if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+
+ numUpsertions, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Upsertions = make([]AlterUserScramCredentialsUpsert, numUpsertions)
+ for i := 0; i < numUpsertions; i++ {
+ r.Upsertions[i] = AlterUserScramCredentialsUpsert{}
+ if r.Upsertions[i].Name, err = pd.getString(); err != nil {
+ return err
+ }
+ mechanism, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+
+ r.Upsertions[i].Mechanism = ScramMechanismType(mechanism)
+ if r.Upsertions[i].Iterations, err = pd.getInt32(); err != nil {
+ return err
+ }
+ if r.Upsertions[i].Salt, err = pd.getBytes(); err != nil {
+ return err
+ }
+ if r.Upsertions[i].saltedPassword, err = pd.getBytes(); err != nil {
+ return err
+ }
+ if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *AlterUserScramCredentialsRequest) key() int16 {
+ return apiKeyAlterUserScramCredentials
+}
+
+func (r *AlterUserScramCredentialsRequest) version() int16 {
+ return r.Version
+}
+
+func (r *AlterUserScramCredentialsRequest) headerVersion() int16 {
+ return 2
+}
+
+func (r *AlterUserScramCredentialsRequest) isValidVersion() bool {
+ return r.Version == 0
+}
+
+func (r *AlterUserScramCredentialsRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *AlterUserScramCredentialsRequest) isFlexibleVersion(version int16) bool {
+ return version >= 0
+}
+
+func (r *AlterUserScramCredentialsRequest) requiredVersion() KafkaVersion {
+ return V2_7_0_0
+}
diff --git a/vendor/github.com/IBM/sarama/alter_user_scram_credentials_response.go b/vendor/github.com/IBM/sarama/alter_user_scram_credentials_response.go
new file mode 100644
index 0000000..229ed54
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/alter_user_scram_credentials_response.go
@@ -0,0 +1,111 @@
+package sarama
+
+import "time"
+
+type AlterUserScramCredentialsResponse struct {
+ Version int16
+
+ ThrottleTime time.Duration
+
+ Results []*AlterUserScramCredentialsResult
+}
+
+func (r *AlterUserScramCredentialsResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+type AlterUserScramCredentialsResult struct {
+ User string
+
+ ErrorCode KError
+ ErrorMessage *string
+}
+
+func (r *AlterUserScramCredentialsResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(r.ThrottleTime)
+ if err := pe.putArrayLength(len(r.Results)); err != nil {
+ return err
+ }
+
+ for _, u := range r.Results {
+ if err := pe.putString(u.User); err != nil {
+ return err
+ }
+ pe.putKError(u.ErrorCode)
+ if err := pe.putNullableString(u.ErrorMessage); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *AlterUserScramCredentialsResponse) decode(pd packetDecoder, version int16) (err error) {
+ if r.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ numResults, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if numResults > 0 {
+ r.Results = make([]*AlterUserScramCredentialsResult, numResults)
+ for i := 0; i < numResults; i++ {
+ r.Results[i] = &AlterUserScramCredentialsResult{}
+ if r.Results[i].User, err = pd.getString(); err != nil {
+ return err
+ }
+
+ r.Results[i].ErrorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if r.Results[i].ErrorMessage, err = pd.getNullableString(); err != nil {
+ return err
+ }
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *AlterUserScramCredentialsResponse) key() int16 {
+ return apiKeyAlterUserScramCredentials
+}
+
+func (r *AlterUserScramCredentialsResponse) version() int16 {
+ return r.Version
+}
+
+func (r *AlterUserScramCredentialsResponse) headerVersion() int16 {
+ return 2
+}
+
+func (r *AlterUserScramCredentialsResponse) isValidVersion() bool {
+ return r.Version == 0
+}
+
+func (r *AlterUserScramCredentialsResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *AlterUserScramCredentialsResponse) isFlexibleVersion(version int16) bool {
+ return version >= 0
+}
+
+func (r *AlterUserScramCredentialsResponse) requiredVersion() KafkaVersion {
+ return V2_7_0_0
+}
+
+func (r *AlterUserScramCredentialsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/api_versions.go b/vendor/github.com/IBM/sarama/api_versions.go
new file mode 100644
index 0000000..e993432
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/api_versions.go
@@ -0,0 +1,84 @@
+package sarama
+
+type apiVersionRange struct {
+ minVersion int16
+ maxVersion int16
+}
+
+type apiVersionMap map[int16]*apiVersionRange
+
+// restrictApiVersion selects the appropriate API version for a given protocol body according to
+// the client and broker version ranges. By default, it selects the maximum version supported by both
+// client and broker, capped by the maximum Kafka version from Config.
+// It then calls setVersion() on the protocol body.
+// If no valid version is found, an error is returned.
+func restrictApiVersion(pb protocolBody, brokerVersions apiVersionMap) error {
+ key := pb.key()
+ // Since message constructors take a Kafka version and select the maximum supported protocol version already, we can
+ // rely on pb.version() being the max version supported for the user-selected Kafka API version.
+ clientMax := pb.version()
+
+ if brokerVersionRange := brokerVersions[key]; brokerVersionRange != nil {
+ // Select the maximum version that both client and server support
+ // Clamp to the client max to respect user preference above broker advertised version range
+ pb.setVersion(min(clientMax, max(min(clientMax, brokerVersionRange.maxVersion), brokerVersionRange.minVersion)))
+ return nil
+ }
+
+ return nil // no version ranges available, no restriction
+}
+
+const (
+ apiKeyProduce = 0
+ apiKeyFetch = 1
+ apiKeyListOffsets = 2
+ apiKeyMetadata = 3
+ apiKeyLeaderAndIsr = 4
+ apiKeyStopReplica = 5
+ apiKeyUpdateMetadata = 6
+ apiKeyControlledShutdown = 7
+ apiKeyOffsetCommit = 8
+ apiKeyOffsetFetch = 9
+ apiKeyFindCoordinator = 10
+ apiKeyJoinGroup = 11
+ apiKeyHeartbeat = 12
+ apiKeyLeaveGroup = 13
+ apiKeySyncGroup = 14
+ apiKeyDescribeGroups = 15
+ apiKeyListGroups = 16
+ apiKeySaslHandshake = 17
+ apiKeyApiVersions = 18
+ apiKeyCreateTopics = 19
+ apiKeyDeleteTopics = 20
+ apiKeyDeleteRecords = 21
+ apiKeyInitProducerId = 22
+ apiKeyOffsetForLeaderEpoch = 23
+ apiKeyAddPartitionsToTxn = 24
+ apiKeyAddOffsetsToTxn = 25
+ apiKeyEndTxn = 26
+ apiKeyWriteTxnMarkers = 27
+ apiKeyTxnOffsetCommit = 28
+ apiKeyDescribeAcls = 29
+ apiKeyCreateAcls = 30
+ apiKeyDeleteAcls = 31
+ apiKeyDescribeConfigs = 32
+ apiKeyAlterConfigs = 33
+ apiKeyAlterReplicaLogDirs = 34
+ apiKeyDescribeLogDirs = 35
+ apiKeySASLAuth = 36
+ apiKeyCreatePartitions = 37
+ apiKeyCreateDelegationToken = 38
+ apiKeyRenewDelegationToken = 39
+ apiKeyExpireDelegationToken = 40
+ apiKeyDescribeDelegationToken = 41
+ apiKeyDeleteGroups = 42
+ apiKeyElectLeaders = 43
+ apiKeyIncrementalAlterConfigs = 44
+ apiKeyAlterPartitionReassignments = 45
+ apiKeyListPartitionReassignments = 46
+ apiKeyOffsetDelete = 47
+ apiKeyDescribeClientQuotas = 48
+ apiKeyAlterClientQuotas = 49
+ apiKeyDescribeUserScramCredentials = 50
+ apiKeyAlterUserScramCredentials = 51
+)
diff --git a/vendor/github.com/IBM/sarama/api_versions_request.go b/vendor/github.com/IBM/sarama/api_versions_request.go
new file mode 100644
index 0000000..593b0f0
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/api_versions_request.go
@@ -0,0 +1,87 @@
+package sarama
+
+const defaultClientSoftwareName = "sarama"
+
+type ApiVersionsRequest struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // ClientSoftwareName contains the name of the client.
+ ClientSoftwareName string
+ // ClientSoftwareVersion contains the version of the client.
+ ClientSoftwareVersion string
+}
+
+func (r *ApiVersionsRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *ApiVersionsRequest) encode(pe packetEncoder) (err error) {
+ if r.Version >= 3 {
+ if err := pe.putString(r.ClientSoftwareName); err != nil {
+ return err
+ }
+ if err := pe.putString(r.ClientSoftwareVersion); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ return nil
+}
+
+func (r *ApiVersionsRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ if r.Version >= 3 {
+ if r.ClientSoftwareName, err = pd.getString(); err != nil {
+ return err
+ }
+ if r.ClientSoftwareVersion, err = pd.getString(); err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *ApiVersionsRequest) key() int16 {
+ return apiKeyApiVersions
+}
+
+func (r *ApiVersionsRequest) version() int16 {
+ return r.Version
+}
+
+func (r *ApiVersionsRequest) headerVersion() int16 {
+ if r.Version >= 3 {
+ return 2
+ }
+ return 1
+}
+
+func (r *ApiVersionsRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 3
+}
+
+func (r *ApiVersionsRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *ApiVersionsRequest) isFlexibleVersion(version int16) bool {
+ return version >= 3
+}
+
+func (r *ApiVersionsRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 3:
+ return V2_4_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_10_0_0
+ default:
+ return V2_4_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/api_versions_response.go b/vendor/github.com/IBM/sarama/api_versions_response.go
new file mode 100644
index 0000000..3f3c11a
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/api_versions_response.go
@@ -0,0 +1,171 @@
+package sarama
+
+import (
+ "time"
+)
+
+// ApiVersionsResponseKey contains the APIs supported by the broker.
+type ApiVersionsResponseKey struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // ApiKey contains the API index.
+ ApiKey int16
+ // MinVersion contains the minimum supported version, inclusive.
+ MinVersion int16
+ // MaxVersion contains the maximum supported version, inclusive.
+ MaxVersion int16
+}
+
+func (a *ApiVersionsResponseKey) encode(pe packetEncoder, version int16) (err error) {
+ a.Version = version
+ pe.putInt16(a.ApiKey)
+
+ pe.putInt16(a.MinVersion)
+
+ pe.putInt16(a.MaxVersion)
+
+ if version >= 3 {
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ return nil
+}
+
+func (a *ApiVersionsResponseKey) decode(pd packetDecoder, version int16) (err error) {
+ a.Version = version
+ if a.ApiKey, err = pd.getInt16(); err != nil {
+ return err
+ }
+
+ if a.MinVersion, err = pd.getInt16(); err != nil {
+ return err
+ }
+
+ if a.MaxVersion, err = pd.getInt16(); err != nil {
+ return err
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+type ApiVersionsResponse struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // ErrorCode contains the top-level error code.
+ ErrorCode int16
+ // ApiKeys contains the APIs supported by the broker.
+ ApiKeys []ApiVersionsResponseKey
+ // ThrottleTimeMs contains the duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota.
+ ThrottleTimeMs int32
+}
+
+func (r *ApiVersionsResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *ApiVersionsResponse) encode(pe packetEncoder) (err error) {
+ pe.putInt16(r.ErrorCode)
+
+ if err := pe.putArrayLength(len(r.ApiKeys)); err != nil {
+ return err
+ }
+ for _, block := range r.ApiKeys {
+ if err := block.encode(pe, r.Version); err != nil {
+ return err
+ }
+ }
+
+ if r.Version >= 1 {
+ pe.putInt32(r.ThrottleTimeMs)
+ }
+
+ if r.Version >= 3 {
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ return nil
+}
+
+func (r *ApiVersionsResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ if r.ErrorCode, err = pd.getInt16(); err != nil {
+ return err
+ }
+
+ // KIP-511: if broker didn't understand the ApiVersionsRequest version then
+ // it replies with a V0 non-flexible ApiVersionResponse where its supported
+ // ApiVersionsRequest version is available in ApiKeys
+ if r.ErrorCode == int16(ErrUnsupportedVersion) {
+ // drop version to 0 and to revert packageDecoder to non-flexible for remaining decoding
+ r.Version = 0
+ pd = downgradeFlexibleDecoder(pd)
+ }
+
+ numApiKeys, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ r.ApiKeys = make([]ApiVersionsResponseKey, numApiKeys)
+ for i := 0; i < numApiKeys; i++ {
+ var block ApiVersionsResponseKey
+ if err = block.decode(pd, r.Version); err != nil {
+ return err
+ }
+ r.ApiKeys[i] = block
+ }
+
+ if r.Version >= 1 {
+ if r.ThrottleTimeMs, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *ApiVersionsResponse) key() int16 {
+ return apiKeyApiVersions
+}
+
+func (r *ApiVersionsResponse) version() int16 {
+ return r.Version
+}
+
+func (r *ApiVersionsResponse) headerVersion() int16 {
+ // ApiVersionsResponse always includes a v0 header.
+ // See KIP-511 for details
+ return 0
+}
+
+func (r *ApiVersionsResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 3
+}
+
+func (r *ApiVersionsResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *ApiVersionsResponse) isFlexibleVersion(version int16) bool {
+ return version >= 3
+}
+
+func (r *ApiVersionsResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 3:
+ return V2_4_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_10_0_0
+ default:
+ return V2_4_0_0
+ }
+}
+
+func (r *ApiVersionsResponse) throttleTime() time.Duration {
+ return time.Duration(r.ThrottleTimeMs) * time.Millisecond
+}
diff --git a/vendor/github.com/IBM/sarama/async_producer.go b/vendor/github.com/IBM/sarama/async_producer.go
new file mode 100644
index 0000000..e34eed5
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/async_producer.go
@@ -0,0 +1,1427 @@
+package sarama
+
+import (
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "math"
+ "sync"
+ "time"
+
+ "github.com/eapache/go-resiliency/breaker"
+ "github.com/eapache/queue"
+ "github.com/rcrowley/go-metrics"
+)
+
+// ErrProducerRetryBufferOverflow is returned when the bridging retry buffer is full and OOM prevention needs to be applied.
+var ErrProducerRetryBufferOverflow = errors.New("retry buffer full: message discarded to prevent buffer overflow")
+
+const (
+ // minFunctionalRetryBufferLength defines the minimum number of messages the retry buffer must support.
+ // If Producer.Retry.MaxBufferLength is set to a non-zero value below this limit, it will be adjusted to this value.
+ // This ensures the retry buffer remains functional under typical workloads.
+ minFunctionalRetryBufferLength = 4 * 1024
+ // minFunctionalRetryBufferBytes defines the minimum total byte size the retry buffer must support.
+ // If Producer.Retry.MaxBufferBytes is set to a non-zero value below this limit, it will be adjusted to this value.
+ // A 32 MB lower limit ensures sufficient capacity for retrying larger messages without exhausting resources.
+ minFunctionalRetryBufferBytes = 32 * 1024 * 1024
+)
+
+// AsyncProducer publishes Kafka messages using a non-blocking API. It routes messages
+// to the correct broker for the provided topic-partition, refreshing metadata as appropriate,
+// and parses responses for errors. You must read from the Errors() channel or the
+// producer will deadlock. You must call Close() or AsyncClose() on a producer to avoid
+// leaks and message lost: it will not be garbage-collected automatically when it passes
+// out of scope and buffered messages may not be flushed.
+type AsyncProducer interface {
+ // AsyncClose triggers a shutdown of the producer. The shutdown has completed
+ // when both the Errors and Successes channels have been closed. When calling
+ // AsyncClose, you *must* continue to read from those channels in order to
+ // drain the results of any messages in flight.
+ AsyncClose()
+
+ // Close shuts down the producer and waits for any buffered messages to be
+ // flushed. You must call this function before a producer object passes out of
+ // scope, as it may otherwise leak memory. You must call this before process
+ // shutting down, or you may lose messages. You must call this before calling
+ // Close on the underlying client.
+ Close() error
+
+ // Input is the input channel for the user to write messages to that they
+ // wish to send.
+ Input() chan<- *ProducerMessage
+
+ // Successes is the success output channel back to the user when Return.Successes is
+ // enabled. If Return.Successes is true, you MUST read from this channel or the
+ // Producer will deadlock. It is suggested that you send and read messages
+ // together in a single select statement.
+ Successes() <-chan *ProducerMessage
+
+ // Errors is the error output channel back to the user. You MUST read from this
+ // channel or the Producer will deadlock when the channel is full. Alternatively,
+ // you can set Producer.Return.Errors in your config to false, which prevents
+ // errors to be returned.
+ Errors() <-chan *ProducerError
+
+ // IsTransactional return true when current producer is transactional.
+ IsTransactional() bool
+
+ // TxnStatus return current producer transaction status.
+ TxnStatus() ProducerTxnStatusFlag
+
+ // BeginTxn mark current transaction as ready.
+ BeginTxn() error
+
+ // CommitTxn commit current transaction.
+ CommitTxn() error
+
+ // AbortTxn abort current transaction.
+ AbortTxn() error
+
+ // AddOffsetsToTxn add associated offsets to current transaction.
+ AddOffsetsToTxn(offsets map[string][]*PartitionOffsetMetadata, groupId string) error
+
+ // AddMessageToTxn add message offsets to current transaction.
+ AddMessageToTxn(msg *ConsumerMessage, groupId string, metadata *string) error
+}
+
+type asyncProducer struct {
+ client Client
+ conf *Config
+
+ errors chan *ProducerError
+ input, successes, retries chan *ProducerMessage
+ inFlight sync.WaitGroup
+
+ brokers map[*Broker]*brokerProducer
+ brokerRefs map[*brokerProducer]int
+ brokerLock sync.Mutex
+
+ txnmgr *transactionManager
+ txLock sync.Mutex
+
+ metricsRegistry metrics.Registry
+}
+
+// NewAsyncProducer creates a new AsyncProducer using the given broker addresses and configuration.
+func NewAsyncProducer(addrs []string, conf *Config) (AsyncProducer, error) {
+ client, err := NewClient(addrs, conf)
+ if err != nil {
+ return nil, err
+ }
+ return newAsyncProducer(client)
+}
+
+// NewAsyncProducerFromClient creates a new Producer using the given client. It is still
+// necessary to call Close() on the underlying client when shutting down this producer.
+func NewAsyncProducerFromClient(client Client) (AsyncProducer, error) {
+ // For clients passed in by the client, ensure we don't
+ // call Close() on it.
+ cli := &nopCloserClient{client}
+ return newAsyncProducer(cli)
+}
+
+func newAsyncProducer(client Client) (AsyncProducer, error) {
+ // Check that we are not dealing with a closed Client before processing any other arguments
+ if client.Closed() {
+ return nil, ErrClosedClient
+ }
+
+ txnmgr, err := newTransactionManager(client.Config(), client)
+ if err != nil {
+ return nil, err
+ }
+
+ p := &asyncProducer{
+ client: client,
+ conf: client.Config(),
+ errors: make(chan *ProducerError),
+ input: make(chan *ProducerMessage),
+ successes: make(chan *ProducerMessage),
+ retries: make(chan *ProducerMessage),
+ brokers: make(map[*Broker]*brokerProducer),
+ brokerRefs: make(map[*brokerProducer]int),
+ txnmgr: txnmgr,
+ metricsRegistry: newCleanupRegistry(client.Config().MetricRegistry),
+ }
+
+ // launch our singleton dispatchers
+ go withRecover(p.dispatcher)
+ go withRecover(p.retryHandler)
+
+ return p, nil
+}
+
+type flagSet int8
+
+const (
+ syn flagSet = 1 << iota // first message from partitionProducer to brokerProducer
+ fin // final message from partitionProducer to brokerProducer and back
+ shutdown // start the shutdown process
+ endtxn // endtxn
+ committxn // endtxn
+ aborttxn // endtxn
+)
+
+// ProducerMessage is the collection of elements passed to the Producer in order to send a message.
+type ProducerMessage struct {
+ Topic string // The Kafka topic for this message.
+ // The partitioning key for this message. Pre-existing Encoders include
+ // StringEncoder and ByteEncoder.
+ Key Encoder
+ // The actual message to store in Kafka. Pre-existing Encoders include
+ // StringEncoder and ByteEncoder.
+ Value Encoder
+
+ // The headers are key-value pairs that are transparently passed
+ // by Kafka between producers and consumers.
+ Headers []RecordHeader
+
+ // This field is used to hold arbitrary data you wish to include so it
+ // will be available when receiving on the Successes and Errors channels.
+ // Sarama completely ignores this field and is only to be used for
+ // pass-through data.
+ Metadata interface{}
+
+ // Below this point are filled in by the producer as the message is processed
+
+ // Offset is the offset of the message stored on the broker. This is only
+ // guaranteed to be defined if the message was successfully delivered and
+ // RequiredAcks is not NoResponse.
+ Offset int64
+ // Partition is the partition that the message was sent to. This is only
+ // guaranteed to be defined if the message was successfully delivered.
+ Partition int32
+ // Timestamp can vary in behavior depending on broker configuration, being
+ // in either one of the CreateTime or LogAppendTime modes (default CreateTime),
+ // and requiring version at least 0.10.0.
+ //
+ // When configured to CreateTime, the timestamp is specified by the producer
+ // either by explicitly setting this field, or when the message is added
+ // to a produce set.
+ //
+ // When configured to LogAppendTime, the timestamp assigned to the message
+ // by the broker. This is only guaranteed to be defined if the message was
+ // successfully delivered and RequiredAcks is not NoResponse.
+ Timestamp time.Time
+
+ retries int
+ flags flagSet
+ expectation chan *ProducerError
+ sequenceNumber int32
+ producerEpoch int16
+ hasSequence bool
+}
+
+const producerMessageOverhead = 26 // the metadata overhead of CRC, flags, etc.
+
+func (m *ProducerMessage) ByteSize(version int) int {
+ var size int
+ if version >= 2 {
+ size = maximumRecordOverhead
+ for _, h := range m.Headers {
+ size += len(h.Key) + len(h.Value) + 2*binary.MaxVarintLen32
+ }
+ } else {
+ size = producerMessageOverhead
+ }
+ if m.Key != nil {
+ size += m.Key.Length()
+ }
+ if m.Value != nil {
+ size += m.Value.Length()
+ }
+ return size
+}
+
+func (m *ProducerMessage) clear() {
+ m.flags = 0
+ m.retries = 0
+ m.sequenceNumber = 0
+ m.producerEpoch = 0
+ m.hasSequence = false
+}
+
+// ProducerError is the type of error generated when the producer fails to deliver a message.
+// It contains the original ProducerMessage as well as the actual error value.
+type ProducerError struct {
+ Msg *ProducerMessage
+ Err error
+}
+
+func (pe ProducerError) Error() string {
+ return fmt.Sprintf("kafka: Failed to produce message to topic %s: %s", pe.Msg.Topic, pe.Err)
+}
+
+func (pe ProducerError) Unwrap() error {
+ return pe.Err
+}
+
+// ProducerErrors is a type that wraps a batch of "ProducerError"s and implements the Error interface.
+// It can be returned from the Producer's Close method to avoid the need to manually drain the Errors channel
+// when closing a producer.
+type ProducerErrors []*ProducerError
+
+func (pe ProducerErrors) Error() string {
+ return fmt.Sprintf("kafka: Failed to deliver %d messages.", len(pe))
+}
+
+func (p *asyncProducer) IsTransactional() bool {
+ return p.txnmgr.isTransactional()
+}
+
+func (p *asyncProducer) AddMessageToTxn(msg *ConsumerMessage, groupId string, metadata *string) error {
+ offsets := make(map[string][]*PartitionOffsetMetadata)
+ offsets[msg.Topic] = []*PartitionOffsetMetadata{
+ {
+ Partition: msg.Partition,
+ Offset: msg.Offset + 1,
+ Metadata: metadata,
+ },
+ }
+ return p.AddOffsetsToTxn(offsets, groupId)
+}
+
+func (p *asyncProducer) AddOffsetsToTxn(offsets map[string][]*PartitionOffsetMetadata, groupId string) error {
+ p.txLock.Lock()
+ defer p.txLock.Unlock()
+
+ if !p.IsTransactional() {
+ DebugLogger.Printf("producer/txnmgr [%s] attempt to call AddOffsetsToTxn on a non-transactional producer\n", p.txnmgr.transactionalID)
+ return ErrNonTransactedProducer
+ }
+
+ DebugLogger.Printf("producer/txnmgr [%s] add offsets to transaction\n", p.txnmgr.transactionalID)
+ return p.txnmgr.addOffsetsToTxn(offsets, groupId)
+}
+
+func (p *asyncProducer) TxnStatus() ProducerTxnStatusFlag {
+ return p.txnmgr.currentTxnStatus()
+}
+
+func (p *asyncProducer) BeginTxn() error {
+ p.txLock.Lock()
+ defer p.txLock.Unlock()
+
+ if !p.IsTransactional() {
+ DebugLogger.Println("producer/txnmgr attempt to call BeginTxn on a non-transactional producer")
+ return ErrNonTransactedProducer
+ }
+
+ return p.txnmgr.transitionTo(ProducerTxnFlagInTransaction, nil)
+}
+
+func (p *asyncProducer) CommitTxn() error {
+ p.txLock.Lock()
+ defer p.txLock.Unlock()
+
+ if !p.IsTransactional() {
+ DebugLogger.Printf("producer/txnmgr [%s] attempt to call CommitTxn on a non-transactional producer\n", p.txnmgr.transactionalID)
+ return ErrNonTransactedProducer
+ }
+
+ DebugLogger.Printf("producer/txnmgr [%s] committing transaction\n", p.txnmgr.transactionalID)
+ err := p.finishTransaction(true)
+ if err != nil {
+ return err
+ }
+ DebugLogger.Printf("producer/txnmgr [%s] transaction committed\n", p.txnmgr.transactionalID)
+ return nil
+}
+
+func (p *asyncProducer) AbortTxn() error {
+ p.txLock.Lock()
+ defer p.txLock.Unlock()
+
+ if !p.IsTransactional() {
+ DebugLogger.Printf("producer/txnmgr [%s] attempt to call AbortTxn on a non-transactional producer\n", p.txnmgr.transactionalID)
+ return ErrNonTransactedProducer
+ }
+ DebugLogger.Printf("producer/txnmgr [%s] aborting transaction\n", p.txnmgr.transactionalID)
+ err := p.finishTransaction(false)
+ if err != nil {
+ return err
+ }
+ DebugLogger.Printf("producer/txnmgr [%s] transaction aborted\n", p.txnmgr.transactionalID)
+ return nil
+}
+
+func (p *asyncProducer) finishTransaction(commit bool) error {
+ p.inFlight.Add(1)
+ if commit {
+ p.input <- &ProducerMessage{flags: endtxn | committxn}
+ } else {
+ p.input <- &ProducerMessage{flags: endtxn | aborttxn}
+ }
+ p.inFlight.Wait()
+ return p.txnmgr.finishTransaction(commit)
+}
+
+func (p *asyncProducer) Errors() <-chan *ProducerError {
+ return p.errors
+}
+
+func (p *asyncProducer) Successes() <-chan *ProducerMessage {
+ return p.successes
+}
+
+func (p *asyncProducer) Input() chan<- *ProducerMessage {
+ return p.input
+}
+
+func (p *asyncProducer) Close() error {
+ p.AsyncClose()
+
+ if p.conf.Producer.Return.Successes {
+ go withRecover(func() {
+ for range p.successes {
+ }
+ })
+ }
+
+ var pErrs ProducerErrors
+ if p.conf.Producer.Return.Errors {
+ for event := range p.errors {
+ pErrs = append(pErrs, event)
+ }
+ } else {
+ <-p.errors
+ }
+
+ if len(pErrs) > 0 {
+ return pErrs
+ }
+ return nil
+}
+
+func (p *asyncProducer) AsyncClose() {
+ go withRecover(p.shutdown)
+}
+
+// singleton
+// dispatches messages by topic
+func (p *asyncProducer) dispatcher() {
+ handlers := make(map[string]chan<- *ProducerMessage)
+ shuttingDown := false
+
+ for msg := range p.input {
+ if msg == nil {
+ Logger.Println("Something tried to send a nil message, it was ignored.")
+ continue
+ }
+
+ if msg.flags&endtxn != 0 {
+ var err error
+ if msg.flags&committxn != 0 {
+ err = p.txnmgr.transitionTo(ProducerTxnFlagEndTransaction|ProducerTxnFlagCommittingTransaction, nil)
+ } else {
+ err = p.txnmgr.transitionTo(ProducerTxnFlagEndTransaction|ProducerTxnFlagAbortingTransaction, nil)
+ }
+ if err != nil {
+ Logger.Printf("producer/txnmgr unable to end transaction %s", err)
+ }
+ p.inFlight.Done()
+ continue
+ }
+
+ if msg.flags&shutdown != 0 {
+ shuttingDown = true
+ p.inFlight.Done()
+ continue
+ }
+
+ if msg.retries == 0 {
+ if shuttingDown {
+ // we can't just call returnError here because that decrements the wait group,
+ // which hasn't been incremented yet for this message, and shouldn't be
+ pErr := &ProducerError{Msg: msg, Err: ErrShuttingDown}
+ if p.conf.Producer.Return.Errors {
+ p.errors <- pErr
+ } else {
+ Logger.Println(pErr)
+ }
+ continue
+ }
+ p.inFlight.Add(1)
+ // Ignore retried msg, there are already in txn.
+ // Can't produce new record when transaction is not started.
+ if p.IsTransactional() && p.txnmgr.currentTxnStatus()&ProducerTxnFlagInTransaction == 0 {
+ Logger.Printf("attempt to send message when transaction is not started or is in ending state, got %d, expect %d\n", p.txnmgr.currentTxnStatus(), ProducerTxnFlagInTransaction)
+ p.returnError(msg, ErrTransactionNotReady)
+ continue
+ }
+ }
+
+ for _, interceptor := range p.conf.Producer.Interceptors {
+ msg.safelyApplyInterceptor(interceptor)
+ }
+
+ version := 1
+ if p.conf.Version.IsAtLeast(V0_11_0_0) {
+ version = 2
+ } else if msg.Headers != nil {
+ p.returnError(msg, ConfigurationError("Producing headers requires Kafka at least v0.11"))
+ continue
+ }
+
+ size := msg.ByteSize(version)
+ if size > p.conf.Producer.MaxMessageBytes {
+ p.returnError(msg, ConfigurationError(fmt.Sprintf("Attempt to produce message larger than configured Producer.MaxMessageBytes: %d > %d", size, p.conf.Producer.MaxMessageBytes)))
+ continue
+ }
+
+ handler := handlers[msg.Topic]
+ if handler == nil {
+ handler = p.newTopicProducer(msg.Topic)
+ handlers[msg.Topic] = handler
+ }
+
+ handler <- msg
+ }
+
+ for _, handler := range handlers {
+ close(handler)
+ }
+}
+
+// one per topic
+// partitions messages, then dispatches them by partition
+type topicProducer struct {
+ parent *asyncProducer
+ topic string
+ input <-chan *ProducerMessage
+
+ breaker *breaker.Breaker
+ handlers map[int32]chan<- *ProducerMessage
+ partitioner Partitioner
+}
+
+func (p *asyncProducer) newTopicProducer(topic string) chan<- *ProducerMessage {
+ input := make(chan *ProducerMessage, p.conf.ChannelBufferSize)
+ tp := &topicProducer{
+ parent: p,
+ topic: topic,
+ input: input,
+ breaker: breaker.New(3, 1, 10*time.Second),
+ handlers: make(map[int32]chan<- *ProducerMessage),
+ partitioner: p.conf.Producer.Partitioner(topic),
+ }
+ go withRecover(tp.dispatch)
+ return input
+}
+
+func (tp *topicProducer) dispatch() {
+ for msg := range tp.input {
+ if msg.retries == 0 {
+ if err := tp.partitionMessage(msg); err != nil {
+ tp.parent.returnError(msg, err)
+ continue
+ }
+ }
+
+ handler := tp.handlers[msg.Partition]
+ if handler == nil {
+ handler = tp.parent.newPartitionProducer(msg.Topic, msg.Partition)
+ tp.handlers[msg.Partition] = handler
+ }
+
+ handler <- msg
+ }
+
+ for _, handler := range tp.handlers {
+ close(handler)
+ }
+}
+
+func (tp *topicProducer) partitionMessage(msg *ProducerMessage) error {
+ var partitions []int32
+
+ err := tp.breaker.Run(func() (err error) {
+ requiresConsistency := false
+ if ep, ok := tp.partitioner.(DynamicConsistencyPartitioner); ok {
+ requiresConsistency = ep.MessageRequiresConsistency(msg)
+ } else {
+ requiresConsistency = tp.partitioner.RequiresConsistency()
+ }
+
+ if requiresConsistency {
+ partitions, err = tp.parent.client.Partitions(msg.Topic)
+ } else {
+ partitions, err = tp.parent.client.WritablePartitions(msg.Topic)
+ }
+ return
+ })
+ if err != nil {
+ return err
+ }
+
+ numPartitions := int32(len(partitions))
+
+ if numPartitions == 0 {
+ return ErrLeaderNotAvailable
+ }
+
+ choice, err := tp.partitioner.Partition(msg, numPartitions)
+
+ if err != nil {
+ return err
+ } else if choice < 0 || choice >= numPartitions {
+ return ErrInvalidPartition
+ }
+
+ msg.Partition = partitions[choice]
+
+ return nil
+}
+
+// one per partition per topic
+// dispatches messages to the appropriate broker
+// also responsible for maintaining message order during retries
+type partitionProducer struct {
+ parent *asyncProducer
+ topic string
+ partition int32
+ input <-chan *ProducerMessage
+
+ leader *Broker
+ breaker *breaker.Breaker
+ brokerProducer *brokerProducer
+
+ // highWatermark tracks the "current" retry level, which is the only one where we actually let messages through,
+ // all other messages get buffered in retryState[msg.retries].buf to preserve ordering
+ // retryState[msg.retries].expectChaser simply tracks whether we've seen a fin message for a given level (and
+ // therefore whether our buffer is complete and safe to flush)
+ highWatermark int
+ retryState []partitionRetryState
+}
+
+type partitionRetryState struct {
+ buf []*ProducerMessage
+ expectChaser bool
+}
+
+func (p *asyncProducer) newPartitionProducer(topic string, partition int32) chan<- *ProducerMessage {
+ input := make(chan *ProducerMessage, p.conf.ChannelBufferSize)
+ pp := &partitionProducer{
+ parent: p,
+ topic: topic,
+ partition: partition,
+ input: input,
+
+ breaker: breaker.New(3, 1, 10*time.Second),
+ retryState: make([]partitionRetryState, p.conf.Producer.Retry.Max+1),
+ }
+ go withRecover(pp.dispatch)
+ return input
+}
+
+func (pp *partitionProducer) backoff(retries int) {
+ var backoff time.Duration
+ if pp.parent.conf.Producer.Retry.BackoffFunc != nil {
+ maxRetries := pp.parent.conf.Producer.Retry.Max
+ backoff = pp.parent.conf.Producer.Retry.BackoffFunc(retries, maxRetries)
+ } else {
+ backoff = pp.parent.conf.Producer.Retry.Backoff
+ }
+ if backoff > 0 {
+ time.Sleep(backoff)
+ }
+}
+
+func (pp *partitionProducer) updateLeaderIfBrokerProducerIsNil(msg *ProducerMessage) error {
+ if pp.brokerProducer == nil {
+ if err := pp.updateLeader(); err != nil {
+ pp.parent.returnError(msg, err)
+ pp.backoff(msg.retries)
+ return err
+ }
+ Logger.Printf("producer/leader/%s/%d selected broker %d\n", pp.topic, pp.partition, pp.leader.ID())
+ }
+ return nil
+}
+
+func (pp *partitionProducer) dispatch() {
+ // try to prefetch the leader; if this doesn't work, we'll do a proper call to `updateLeader`
+ // on the first message
+ pp.leader, _ = pp.parent.client.Leader(pp.topic, pp.partition)
+ if pp.leader != nil {
+ pp.brokerProducer = pp.parent.getBrokerProducer(pp.leader)
+ pp.parent.inFlight.Add(1) // we're generating a syn message; track it so we don't shut down while it's still inflight
+ pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn}
+ }
+
+ defer func() {
+ if pp.brokerProducer != nil {
+ pp.parent.unrefBrokerProducer(pp.leader, pp.brokerProducer)
+ }
+ }()
+
+ for msg := range pp.input {
+ if pp.brokerProducer != nil && pp.brokerProducer.abandoned != nil {
+ select {
+ case <-pp.brokerProducer.abandoned:
+ // a message on the abandoned channel means that our current broker selection is out of date
+ Logger.Printf("producer/leader/%s/%d abandoning broker %d\n", pp.topic, pp.partition, pp.leader.ID())
+ pp.parent.unrefBrokerProducer(pp.leader, pp.brokerProducer)
+ pp.brokerProducer = nil
+ time.Sleep(pp.parent.conf.Producer.Retry.Backoff)
+ default:
+ // producer connection is still open.
+ }
+ }
+
+ if msg.retries > pp.highWatermark {
+ if err := pp.updateLeaderIfBrokerProducerIsNil(msg); err != nil {
+ continue
+ }
+ // a new, higher, retry level; handle it and then back off
+ pp.newHighWatermark(msg.retries)
+ pp.backoff(msg.retries)
+ } else if pp.highWatermark > 0 {
+ // we are retrying something (else highWatermark would be 0) but this message is not a *new* retry level
+ if msg.retries < pp.highWatermark {
+ // in fact this message is not even the current retry level, so buffer it for now (unless it's a just a fin)
+ if msg.flags&fin == fin {
+ pp.retryState[msg.retries].expectChaser = false
+ pp.parent.inFlight.Done() // this fin is now handled and will be garbage collected
+ } else {
+ pp.retryState[msg.retries].buf = append(pp.retryState[msg.retries].buf, msg)
+ }
+ continue
+ } else if msg.flags&fin == fin {
+ // this message is of the current retry level (msg.retries == highWatermark) and the fin flag is set,
+ // meaning this retry level is done and we can go down (at least) one level and flush that
+ pp.retryState[pp.highWatermark].expectChaser = false
+ pp.flushRetryBuffers()
+ pp.parent.inFlight.Done() // this fin is now handled and will be garbage collected
+ continue
+ }
+ }
+
+ // if we made it this far then the current msg contains real data, and can be sent to the next goroutine
+ // without breaking any of our ordering guarantees
+ if err := pp.updateLeaderIfBrokerProducerIsNil(msg); err != nil {
+ continue
+ }
+
+ // Now that we know we have a broker to actually try and send this message to, generate the sequence
+ // number for it.
+ // All messages being retried (sent or not) have already had their retry count updated
+ // Also, ignore "special" syn/fin messages used to sync the brokerProducer and the topicProducer.
+ if pp.parent.conf.Producer.Idempotent && msg.retries == 0 && msg.flags == 0 {
+ msg.sequenceNumber, msg.producerEpoch = pp.parent.txnmgr.getAndIncrementSequenceNumber(msg.Topic, msg.Partition)
+ msg.hasSequence = true
+ }
+
+ if pp.parent.IsTransactional() {
+ pp.parent.txnmgr.maybeAddPartitionToCurrentTxn(pp.topic, pp.partition)
+ }
+
+ pp.brokerProducer.input <- msg
+ }
+}
+
+func (pp *partitionProducer) newHighWatermark(hwm int) {
+ Logger.Printf("producer/leader/%s/%d state change to [retrying-%d]\n", pp.topic, pp.partition, hwm)
+ pp.highWatermark = hwm
+
+ // send off a fin so that we know when everything "in between" has made it
+ // back to us and we can safely flush the backlog (otherwise we risk re-ordering messages)
+ pp.retryState[pp.highWatermark].expectChaser = true
+ pp.parent.inFlight.Add(1) // we're generating a fin message; track it so we don't shut down while it's still inflight
+ pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: fin, retries: pp.highWatermark - 1}
+
+ // a new HWM means that our current broker selection is out of date
+ Logger.Printf("producer/leader/%s/%d abandoning broker %d\n", pp.topic, pp.partition, pp.leader.ID())
+ pp.parent.unrefBrokerProducer(pp.leader, pp.brokerProducer)
+ pp.brokerProducer = nil
+}
+
+func (pp *partitionProducer) flushRetryBuffers() {
+ Logger.Printf("producer/leader/%s/%d state change to [flushing-%d]\n", pp.topic, pp.partition, pp.highWatermark)
+ for {
+ pp.highWatermark--
+
+ if pp.brokerProducer == nil {
+ if err := pp.updateLeader(); err != nil {
+ pp.parent.returnErrors(pp.retryState[pp.highWatermark].buf, err)
+ goto flushDone
+ }
+ Logger.Printf("producer/leader/%s/%d selected broker %d\n", pp.topic, pp.partition, pp.leader.ID())
+ }
+
+ for _, msg := range pp.retryState[pp.highWatermark].buf {
+ pp.brokerProducer.input <- msg
+ }
+
+ flushDone:
+ pp.retryState[pp.highWatermark].buf = nil
+ if pp.retryState[pp.highWatermark].expectChaser {
+ Logger.Printf("producer/leader/%s/%d state change to [retrying-%d]\n", pp.topic, pp.partition, pp.highWatermark)
+ break
+ } else if pp.highWatermark == 0 {
+ Logger.Printf("producer/leader/%s/%d state change to [normal]\n", pp.topic, pp.partition)
+ break
+ }
+ }
+}
+
+func (pp *partitionProducer) updateLeader() error {
+ return pp.breaker.Run(func() (err error) {
+ if err = pp.parent.client.RefreshMetadata(pp.topic); err != nil {
+ return err
+ }
+
+ if pp.leader, err = pp.parent.client.Leader(pp.topic, pp.partition); err != nil {
+ return err
+ }
+
+ pp.brokerProducer = pp.parent.getBrokerProducer(pp.leader)
+ pp.parent.inFlight.Add(1) // we're generating a syn message; track it so we don't shut down while it's still inflight
+ pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn}
+
+ return nil
+ })
+}
+
+// one per broker; also constructs an associated flusher
+func (p *asyncProducer) newBrokerProducer(broker *Broker) *brokerProducer {
+ var (
+ input = make(chan *ProducerMessage)
+ bridge = make(chan *produceSet)
+ pending = make(chan *brokerProducerResponse)
+ responses = make(chan *brokerProducerResponse)
+ )
+
+ bp := &brokerProducer{
+ parent: p,
+ broker: broker,
+ input: input,
+ output: bridge,
+ responses: responses,
+ buffer: newProduceSet(p),
+ currentRetries: make(map[string]map[int32]error),
+ }
+ go withRecover(bp.run)
+
+ // minimal bridge to make the network response `select`able
+ go withRecover(func() {
+ // Use a wait group to know if we still have in flight requests
+ var wg sync.WaitGroup
+
+ for set := range bridge {
+ request := set.buildRequest()
+
+ // Count the in flight requests to know when we can close the pending channel safely
+ wg.Add(1)
+ // Capture the current set to forward in the callback
+ sendResponse := func(set *produceSet) ProduceCallback {
+ return func(response *ProduceResponse, err error) {
+ // Forward the response to make sure we do not block the responseReceiver
+ pending <- &brokerProducerResponse{
+ set: set,
+ err: err,
+ res: response,
+ }
+ wg.Done()
+ }
+ }(set)
+
+ if p.IsTransactional() {
+ // Add partition to tx before sending current batch
+ err := p.txnmgr.publishTxnPartitions()
+ if err != nil {
+ // Request failed to be sent
+ sendResponse(nil, err)
+ continue
+ }
+ }
+
+ // Use AsyncProduce vs Produce to not block waiting for the response
+ // so that we can pipeline multiple produce requests and achieve higher throughput, see:
+ // https://kafka.apache.org/protocol#protocol_network
+ err := broker.AsyncProduce(request, sendResponse)
+ if err != nil {
+ // Request failed to be sent
+ sendResponse(nil, err)
+ continue
+ }
+ // Callback is not called when using NoResponse
+ if p.conf.Producer.RequiredAcks == NoResponse {
+ // Provide the expected nil response
+ sendResponse(nil, nil)
+ }
+ }
+ // Wait for all in flight requests to close the pending channel safely
+ wg.Wait()
+ close(pending)
+ })
+
+ // In order to avoid a deadlock when closing the broker on network or malformed response error
+ // we use an intermediate channel to buffer and send pending responses in order
+ // This is because the AsyncProduce callback inside the bridge is invoked from the broker
+ // responseReceiver goroutine and closing the broker requires such goroutine to be finished
+ go withRecover(func() {
+ buf := queue.New()
+ for {
+ if buf.Length() == 0 {
+ res, ok := <-pending
+ if !ok {
+ // We are done forwarding the last pending response
+ close(responses)
+ return
+ }
+ buf.Add(res)
+ }
+ // Send the head pending response or buffer another one
+ // so that we never block the callback
+ headRes := buf.Peek().(*brokerProducerResponse)
+ select {
+ case res, ok := <-pending:
+ if !ok {
+ continue
+ }
+ buf.Add(res)
+ continue
+ case responses <- headRes:
+ buf.Remove()
+ continue
+ }
+ }
+ })
+
+ if p.conf.Producer.Retry.Max <= 0 {
+ bp.abandoned = make(chan struct{})
+ }
+
+ return bp
+}
+
+type brokerProducerResponse struct {
+ set *produceSet
+ err error
+ res *ProduceResponse
+}
+
+// groups messages together into appropriately-sized batches for sending to the broker
+// handles state related to retries etc
+type brokerProducer struct {
+ parent *asyncProducer
+ broker *Broker
+
+ input chan *ProducerMessage
+ output chan<- *produceSet
+ responses <-chan *brokerProducerResponse
+ abandoned chan struct{}
+
+ buffer *produceSet
+ timer *time.Timer
+ timerFired bool
+
+ closing error
+ currentRetries map[string]map[int32]error
+}
+
+func (bp *brokerProducer) run() {
+ var output chan<- *produceSet
+ var timerChan <-chan time.Time
+ Logger.Printf("producer/broker/%d starting up\n", bp.broker.ID())
+
+ for {
+ select {
+ case msg, ok := <-bp.input:
+ if !ok {
+ Logger.Printf("producer/broker/%d input chan closed\n", bp.broker.ID())
+ bp.shutdown()
+ return
+ }
+
+ if msg == nil {
+ continue
+ }
+
+ if msg.flags&syn == syn {
+ Logger.Printf("producer/broker/%d state change to [open] on %s/%d\n",
+ bp.broker.ID(), msg.Topic, msg.Partition)
+ if bp.currentRetries[msg.Topic] == nil {
+ bp.currentRetries[msg.Topic] = make(map[int32]error)
+ }
+ bp.currentRetries[msg.Topic][msg.Partition] = nil
+ bp.parent.inFlight.Done()
+ continue
+ }
+
+ if reason := bp.needsRetry(msg); reason != nil {
+ bp.parent.retryMessage(msg, reason)
+
+ if bp.closing == nil && msg.flags&fin == fin {
+ // we were retrying this partition but we can start processing again
+ delete(bp.currentRetries[msg.Topic], msg.Partition)
+ Logger.Printf("producer/broker/%d state change to [closed] on %s/%d\n",
+ bp.broker.ID(), msg.Topic, msg.Partition)
+ }
+
+ continue
+ }
+
+ if msg.flags&fin == fin {
+ // New broker producer that was caught up by the retry loop
+ bp.parent.retryMessage(msg, ErrShuttingDown)
+ DebugLogger.Printf("producer/broker/%d state change to [dying-%d] on %s/%d\n",
+ bp.broker.ID(), msg.retries, msg.Topic, msg.Partition)
+ continue
+ }
+
+ if bp.buffer.wouldOverflow(msg) {
+ Logger.Printf("producer/broker/%d maximum request accumulated, waiting for space\n", bp.broker.ID())
+ if err := bp.waitForSpace(msg, false); err != nil {
+ bp.parent.retryMessage(msg, err)
+ continue
+ }
+ }
+
+ if bp.parent.txnmgr.producerID != noProducerID && bp.buffer.producerEpoch != msg.producerEpoch {
+ // The epoch was reset, need to roll the buffer over
+ Logger.Printf("producer/broker/%d detected epoch rollover, waiting for new buffer\n", bp.broker.ID())
+ if err := bp.waitForSpace(msg, true); err != nil {
+ bp.parent.retryMessage(msg, err)
+ continue
+ }
+ }
+ if err := bp.buffer.add(msg); err != nil {
+ bp.parent.returnError(msg, err)
+ continue
+ }
+
+ if bp.parent.conf.Producer.Flush.Frequency > 0 && bp.timer == nil {
+ bp.timer = time.NewTimer(bp.parent.conf.Producer.Flush.Frequency)
+ timerChan = bp.timer.C
+ }
+ case <-timerChan:
+ bp.timerFired = true
+ case output <- bp.buffer:
+ bp.rollOver()
+ timerChan = nil
+ case response, ok := <-bp.responses:
+ if ok {
+ bp.handleResponse(response)
+ }
+ }
+
+ if bp.timerFired || bp.buffer.readyToFlush() {
+ output = bp.output
+ } else {
+ output = nil
+ }
+ }
+}
+
+func (bp *brokerProducer) shutdown() {
+ for !bp.buffer.empty() {
+ select {
+ case response := <-bp.responses:
+ bp.handleResponse(response)
+ case bp.output <- bp.buffer:
+ bp.rollOver()
+ }
+ }
+ close(bp.output)
+ // Drain responses from the bridge goroutine
+ for response := range bp.responses {
+ bp.handleResponse(response)
+ }
+ // No more brokerProducer related goroutine should be running
+ Logger.Printf("producer/broker/%d shut down\n", bp.broker.ID())
+}
+
+func (bp *brokerProducer) needsRetry(msg *ProducerMessage) error {
+ if bp.closing != nil {
+ return bp.closing
+ }
+
+ return bp.currentRetries[msg.Topic][msg.Partition]
+}
+
+func (bp *brokerProducer) waitForSpace(msg *ProducerMessage, forceRollover bool) error {
+ for {
+ select {
+ case response := <-bp.responses:
+ bp.handleResponse(response)
+ // handling a response can change our state, so re-check some things
+ if reason := bp.needsRetry(msg); reason != nil {
+ return reason
+ } else if !bp.buffer.wouldOverflow(msg) && !forceRollover {
+ return nil
+ }
+ case bp.output <- bp.buffer:
+ bp.rollOver()
+ return nil
+ }
+ }
+}
+
+func (bp *brokerProducer) rollOver() {
+ if bp.timer != nil {
+ bp.timer.Stop()
+ }
+ bp.timer = nil
+ bp.timerFired = false
+ bp.buffer = newProduceSet(bp.parent)
+}
+
+func (bp *brokerProducer) handleResponse(response *brokerProducerResponse) {
+ if response.err != nil {
+ bp.handleError(response.set, response.err)
+ } else {
+ bp.handleSuccess(response.set, response.res)
+ }
+
+ if bp.buffer.empty() {
+ bp.rollOver() // this can happen if the response invalidated our buffer
+ }
+}
+
+func (bp *brokerProducer) handleSuccess(sent *produceSet, response *ProduceResponse) {
+ // we iterate through the blocks in the request set, not the response, so that we notice
+ // if the response is missing a block completely
+ var retryTopics []string
+ sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) {
+ if response == nil {
+ // this only happens when RequiredAcks is NoResponse, so we have to assume success
+ bp.parent.returnSuccesses(pSet.msgs)
+ return
+ }
+
+ block := response.GetBlock(topic, partition)
+ if block == nil {
+ bp.parent.returnErrors(pSet.msgs, ErrIncompleteResponse)
+ return
+ }
+
+ switch block.Err {
+ // Success
+ case ErrNoError:
+ if bp.parent.conf.Version.IsAtLeast(V0_10_0_0) && !block.Timestamp.IsZero() {
+ for _, msg := range pSet.msgs {
+ msg.Timestamp = block.Timestamp
+ }
+ }
+ for i, msg := range pSet.msgs {
+ msg.Offset = block.Offset + int64(i)
+ }
+ bp.parent.returnSuccesses(pSet.msgs)
+ // Duplicate
+ case ErrDuplicateSequenceNumber:
+ bp.parent.returnSuccesses(pSet.msgs)
+ // Retriable errors
+ case ErrInvalidMessage, ErrUnknownTopicOrPartition, ErrLeaderNotAvailable, ErrNotLeaderForPartition,
+ ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend, ErrKafkaStorageError:
+ if bp.parent.conf.Producer.Retry.Max <= 0 {
+ bp.parent.abandonBrokerConnection(bp.broker)
+ bp.parent.returnErrors(pSet.msgs, block.Err)
+ } else {
+ retryTopics = append(retryTopics, topic)
+ }
+ // Other non-retriable errors
+ default:
+ if bp.parent.conf.Producer.Retry.Max <= 0 {
+ bp.parent.abandonBrokerConnection(bp.broker)
+ }
+ bp.parent.returnErrors(pSet.msgs, block.Err)
+ }
+ })
+
+ if len(retryTopics) > 0 {
+ if bp.parent.conf.Producer.Idempotent {
+ err := bp.parent.client.RefreshMetadata(retryTopics...)
+ if err != nil {
+ Logger.Printf("Failed refreshing metadata because of %v\n", err)
+ }
+ }
+
+ sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) {
+ block := response.GetBlock(topic, partition)
+ if block == nil {
+ // handled in the previous "eachPartition" loop
+ return
+ }
+
+ switch block.Err {
+ case ErrInvalidMessage, ErrUnknownTopicOrPartition, ErrLeaderNotAvailable, ErrNotLeaderForPartition,
+ ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend, ErrKafkaStorageError:
+ Logger.Printf("producer/broker/%d state change to [retrying] on %s/%d because %v\n",
+ bp.broker.ID(), topic, partition, block.Err)
+ if bp.currentRetries[topic] == nil {
+ bp.currentRetries[topic] = make(map[int32]error)
+ }
+ bp.currentRetries[topic][partition] = block.Err
+ if bp.parent.conf.Producer.Idempotent {
+ go bp.parent.retryBatch(topic, partition, pSet, block.Err)
+ } else {
+ bp.parent.retryMessages(pSet.msgs, block.Err)
+ }
+ // dropping the following messages has the side effect of incrementing their retry count
+ bp.parent.retryMessages(bp.buffer.dropPartition(topic, partition), block.Err)
+ }
+ })
+ }
+}
+
+func (p *asyncProducer) retryBatch(topic string, partition int32, pSet *partitionSet, kerr KError) {
+ Logger.Printf("Retrying batch for %v-%d because of %s\n", topic, partition, kerr)
+ produceSet := newProduceSet(p)
+ produceSet.msgs[topic] = make(map[int32]*partitionSet)
+ produceSet.msgs[topic][partition] = pSet
+ produceSet.bufferBytes += pSet.bufferBytes
+ produceSet.bufferCount += len(pSet.msgs)
+ for _, msg := range pSet.msgs {
+ if msg.retries >= p.conf.Producer.Retry.Max {
+ p.returnErrors(pSet.msgs, kerr)
+ return
+ }
+ msg.retries++
+ }
+
+ // it's expected that a metadata refresh has been requested prior to calling retryBatch
+ leader, err := p.client.Leader(topic, partition)
+ if err != nil {
+ Logger.Printf("Failed retrying batch for %v-%d because of %v while looking up for new leader\n", topic, partition, err)
+ for _, msg := range pSet.msgs {
+ p.returnError(msg, kerr)
+ }
+ return
+ }
+ bp := p.getBrokerProducer(leader)
+ bp.output <- produceSet
+ p.unrefBrokerProducer(leader, bp)
+}
+
+func (bp *brokerProducer) handleError(sent *produceSet, err error) {
+ var target PacketEncodingError
+ if errors.As(err, &target) {
+ sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) {
+ bp.parent.returnErrors(pSet.msgs, err)
+ })
+ } else {
+ Logger.Printf("producer/broker/%d state change to [closing] because %s\n", bp.broker.ID(), err)
+ bp.parent.abandonBrokerConnection(bp.broker)
+ _ = bp.broker.Close()
+ bp.closing = err
+ sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) {
+ bp.parent.retryMessages(pSet.msgs, err)
+ })
+ bp.buffer.eachPartition(func(topic string, partition int32, pSet *partitionSet) {
+ bp.parent.retryMessages(pSet.msgs, err)
+ })
+ bp.rollOver()
+ }
+}
+
+// singleton
+// effectively a "bridge" between the flushers and the dispatcher in order to avoid deadlock
+// based on https://godoc.org/github.com/eapache/channels#InfiniteChannel
+func (p *asyncProducer) retryHandler() {
+ maxBufferLength := p.conf.Producer.Retry.MaxBufferLength
+ if 0 < maxBufferLength && maxBufferLength < minFunctionalRetryBufferLength {
+ maxBufferLength = minFunctionalRetryBufferLength
+ }
+
+ maxBufferBytes := p.conf.Producer.Retry.MaxBufferBytes
+ if 0 < maxBufferBytes && maxBufferBytes < minFunctionalRetryBufferBytes {
+ maxBufferBytes = minFunctionalRetryBufferBytes
+ }
+
+ version := 1
+ if p.conf.Version.IsAtLeast(V0_11_0_0) {
+ version = 2
+ }
+
+ var currentByteSize int64
+ var msg *ProducerMessage
+ buf := queue.New()
+
+ for {
+ if buf.Length() == 0 {
+ msg = <-p.retries
+ } else {
+ select {
+ case msg = <-p.retries:
+ case p.input <- buf.Peek().(*ProducerMessage):
+ msgToRemove := buf.Remove().(*ProducerMessage)
+ currentByteSize -= int64(msgToRemove.ByteSize(version))
+ continue
+ }
+ }
+
+ if msg == nil {
+ return
+ }
+
+ buf.Add(msg)
+ currentByteSize += int64(msg.ByteSize(version))
+
+ if (maxBufferLength <= 0 || buf.Length() < maxBufferLength) && (maxBufferBytes <= 0 || currentByteSize < maxBufferBytes) {
+ continue
+ }
+
+ msgToHandle := buf.Peek().(*ProducerMessage)
+ if msgToHandle.flags == 0 {
+ select {
+ case p.input <- msgToHandle:
+ buf.Remove()
+ currentByteSize -= int64(msgToHandle.ByteSize(version))
+ default:
+ buf.Remove()
+ currentByteSize -= int64(msgToHandle.ByteSize(version))
+ p.returnError(msgToHandle, ErrProducerRetryBufferOverflow)
+ }
+ }
+ }
+}
+
+// utility functions
+
+func (p *asyncProducer) shutdown() {
+ Logger.Println("Producer shutting down.")
+ p.inFlight.Add(1)
+ p.input <- &ProducerMessage{flags: shutdown}
+
+ p.inFlight.Wait()
+
+ err := p.client.Close()
+ if err != nil {
+ Logger.Println("producer/shutdown failed to close the embedded client:", err)
+ }
+
+ close(p.input)
+ close(p.retries)
+ close(p.errors)
+ close(p.successes)
+
+ p.metricsRegistry.UnregisterAll()
+}
+
+func (p *asyncProducer) bumpIdempotentProducerEpoch() {
+ _, epoch := p.txnmgr.getProducerID()
+ if epoch == math.MaxInt16 {
+ Logger.Println("producer/txnmanager epoch exhausted, requesting new producer ID")
+ txnmgr, err := newTransactionManager(p.conf, p.client)
+ if err != nil {
+ Logger.Println(err)
+ return
+ }
+
+ p.txnmgr = txnmgr
+ } else {
+ p.txnmgr.bumpEpoch()
+ }
+}
+
+func (p *asyncProducer) maybeTransitionToErrorState(err error) error {
+ if errors.Is(err, ErrClusterAuthorizationFailed) ||
+ errors.Is(err, ErrProducerFenced) ||
+ errors.Is(err, ErrUnsupportedVersion) ||
+ errors.Is(err, ErrTransactionalIDAuthorizationFailed) {
+ return p.txnmgr.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, err)
+ }
+ if p.txnmgr.coordinatorSupportsBumpingEpoch && p.txnmgr.currentTxnStatus()&ProducerTxnFlagEndTransaction == 0 {
+ p.txnmgr.epochBumpRequired = true
+ }
+ return p.txnmgr.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagAbortableError, err)
+}
+
+func (p *asyncProducer) returnError(msg *ProducerMessage, err error) {
+ if p.IsTransactional() {
+ _ = p.maybeTransitionToErrorState(err)
+ }
+ // We need to reset the producer ID epoch if we set a sequence number on it, because the broker
+ // will never see a message with this number, so we can never continue the sequence.
+ if !p.IsTransactional() && msg.hasSequence {
+ Logger.Printf("producer/txnmanager rolling over epoch due to publish failure on %s/%d", msg.Topic, msg.Partition)
+ p.bumpIdempotentProducerEpoch()
+ }
+
+ msg.clear()
+ pErr := &ProducerError{Msg: msg, Err: err}
+ if p.conf.Producer.Return.Errors {
+ p.errors <- pErr
+ } else {
+ Logger.Println(pErr)
+ }
+ p.inFlight.Done()
+}
+
+func (p *asyncProducer) returnErrors(batch []*ProducerMessage, err error) {
+ for _, msg := range batch {
+ p.returnError(msg, err)
+ }
+}
+
+func (p *asyncProducer) returnSuccesses(batch []*ProducerMessage) {
+ for _, msg := range batch {
+ if p.conf.Producer.Return.Successes {
+ msg.clear()
+ p.successes <- msg
+ }
+ p.inFlight.Done()
+ }
+}
+
+func (p *asyncProducer) retryMessage(msg *ProducerMessage, err error) {
+ if msg.retries >= p.conf.Producer.Retry.Max {
+ p.returnError(msg, err)
+ } else {
+ msg.retries++
+ p.retries <- msg
+ }
+}
+
+func (p *asyncProducer) retryMessages(batch []*ProducerMessage, err error) {
+ for _, msg := range batch {
+ p.retryMessage(msg, err)
+ }
+}
+
+func (p *asyncProducer) getBrokerProducer(broker *Broker) *brokerProducer {
+ p.brokerLock.Lock()
+ defer p.brokerLock.Unlock()
+
+ bp := p.brokers[broker]
+
+ if bp == nil {
+ bp = p.newBrokerProducer(broker)
+ p.brokers[broker] = bp
+ p.brokerRefs[bp] = 0
+ }
+
+ p.brokerRefs[bp]++
+
+ return bp
+}
+
+func (p *asyncProducer) unrefBrokerProducer(broker *Broker, bp *brokerProducer) {
+ p.brokerLock.Lock()
+ defer p.brokerLock.Unlock()
+
+ p.brokerRefs[bp]--
+ if p.brokerRefs[bp] == 0 {
+ close(bp.input)
+ delete(p.brokerRefs, bp)
+
+ if p.brokers[broker] == bp {
+ delete(p.brokers, broker)
+ }
+ }
+}
+
+func (p *asyncProducer) abandonBrokerConnection(broker *Broker) {
+ p.brokerLock.Lock()
+ defer p.brokerLock.Unlock()
+
+ bc, ok := p.brokers[broker]
+ if ok && bc.abandoned != nil {
+ close(bc.abandoned)
+ }
+
+ delete(p.brokers, broker)
+}
diff --git a/vendor/github.com/IBM/sarama/balance_strategy.go b/vendor/github.com/IBM/sarama/balance_strategy.go
new file mode 100644
index 0000000..59f8948
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/balance_strategy.go
@@ -0,0 +1,1127 @@
+package sarama
+
+import (
+ "container/heap"
+ "errors"
+ "fmt"
+ "maps"
+ "math"
+ "slices"
+ "sort"
+ "strings"
+)
+
+const (
+ // RangeBalanceStrategyName identifies strategies that use the range partition assignment strategy
+ RangeBalanceStrategyName = "range"
+
+ // RoundRobinBalanceStrategyName identifies strategies that use the round-robin partition assignment strategy
+ RoundRobinBalanceStrategyName = "roundrobin"
+
+ // StickyBalanceStrategyName identifies strategies that use the sticky-partition assignment strategy
+ StickyBalanceStrategyName = "sticky"
+
+ defaultGeneration = -1
+)
+
+// BalanceStrategyPlan is the results of any BalanceStrategy.Plan attempt.
+// It contains an allocation of topic/partitions by memberID in the form of
+// a `memberID -> topic -> partitions` map.
+type BalanceStrategyPlan map[string]map[string][]int32
+
+// Add assigns a topic with a number partitions to a member.
+func (p BalanceStrategyPlan) Add(memberID, topic string, partitions ...int32) {
+ if len(partitions) == 0 {
+ return
+ }
+ if _, ok := p[memberID]; !ok {
+ p[memberID] = make(map[string][]int32, 1)
+ }
+ p[memberID][topic] = append(p[memberID][topic], partitions...)
+}
+
+// --------------------------------------------------------------------
+
+// BalanceStrategy is used to balance topics and partitions
+// across members of a consumer group
+type BalanceStrategy interface {
+ // Name uniquely identifies the strategy.
+ Name() string
+
+ // Plan accepts a map of `memberID -> metadata` and a map of `topic -> partitions`
+ // and returns a distribution plan.
+ Plan(members map[string]ConsumerGroupMemberMetadata, topics map[string][]int32) (BalanceStrategyPlan, error)
+
+ // AssignmentData returns the serialized assignment data for the specified
+ // memberID
+ AssignmentData(memberID string, topics map[string][]int32, generationID int32) ([]byte, error)
+}
+
+// --------------------------------------------------------------------
+
+// NewBalanceStrategyRange returns a range balance strategy,
+// which is the default and assigns partitions as ranges to consumer group members.
+// This follows the same logic as
+// https://kafka.apache.org/31/javadoc/org/apache/kafka/clients/consumer/RangeAssignor.html
+//
+// Example with two topics T1 and T2 with six partitions each (0..5) and two members (M1, M2):
+//
+// M1: {T1: [0, 1, 2], T2: [0, 1, 2]}
+// M2: {T1: [3, 4, 5], T2: [3, 4, 5]}
+func NewBalanceStrategyRange() BalanceStrategy {
+ return &balanceStrategy{
+ name: RangeBalanceStrategyName,
+ coreFn: func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32) {
+ partitionsPerConsumer := len(partitions) / len(memberIDs)
+ consumersWithExtraPartition := len(partitions) % len(memberIDs)
+
+ sort.Strings(memberIDs)
+
+ for i, memberID := range memberIDs {
+ min := i*partitionsPerConsumer + int(math.Min(float64(consumersWithExtraPartition), float64(i)))
+ extra := 0
+ if i < consumersWithExtraPartition {
+ extra = 1
+ }
+ max := min + partitionsPerConsumer + extra
+ plan.Add(memberID, topic, partitions[min:max]...)
+ }
+ },
+ }
+}
+
+// Deprecated: use NewBalanceStrategyRange to avoid data race issue
+var BalanceStrategyRange = NewBalanceStrategyRange()
+
+// NewBalanceStrategySticky returns a sticky balance strategy,
+// which assigns partitions to members with an attempt to preserve earlier assignments
+// while maintain a balanced partition distribution.
+// Example with topic T with six partitions (0..5) and two members (M1, M2):
+//
+// M1: {T: [0, 2, 4]}
+// M2: {T: [1, 3, 5]}
+//
+// On reassignment with an additional consumer, you might get an assignment plan like:
+//
+// M1: {T: [0, 2]}
+// M2: {T: [1, 3]}
+// M3: {T: [4, 5]}
+func NewBalanceStrategySticky() BalanceStrategy {
+ return &stickyBalanceStrategy{}
+}
+
+// Deprecated: use NewBalanceStrategySticky to avoid data race issue
+var BalanceStrategySticky = NewBalanceStrategySticky()
+
+// --------------------------------------------------------------------
+
+type balanceStrategy struct {
+ coreFn func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32)
+ name string
+}
+
+// Name implements BalanceStrategy.
+func (s *balanceStrategy) Name() string { return s.name }
+
+// Plan implements BalanceStrategy.
+func (s *balanceStrategy) Plan(members map[string]ConsumerGroupMemberMetadata, topics map[string][]int32) (BalanceStrategyPlan, error) {
+ // Build members by topic map
+ mbt := make(map[string][]string)
+ for memberID, meta := range members {
+ for _, topic := range meta.Topics {
+ mbt[topic] = append(mbt[topic], memberID)
+ }
+ }
+
+ // func to sort and de-duplicate a StringSlice
+ uniq := func(ss sort.StringSlice) []string {
+ if ss.Len() < 2 {
+ return ss
+ }
+ sort.Sort(ss)
+ var i, j int
+ for i = 1; i < ss.Len(); i++ {
+ if ss[i] == ss[j] {
+ continue
+ }
+ j++
+ ss.Swap(i, j)
+ }
+ return ss[:j+1]
+ }
+
+ // Assemble plan
+ plan := make(BalanceStrategyPlan, len(members))
+ for topic, memberIDs := range mbt {
+ s.coreFn(plan, uniq(memberIDs), topic, topics[topic])
+ }
+ return plan, nil
+}
+
+// AssignmentData simple strategies do not require any shared assignment data
+func (s *balanceStrategy) AssignmentData(memberID string, topics map[string][]int32, generationID int32) ([]byte, error) {
+ return nil, nil
+}
+
+type stickyBalanceStrategy struct {
+ movements partitionMovements
+}
+
+// Name implements BalanceStrategy.
+func (s *stickyBalanceStrategy) Name() string { return StickyBalanceStrategyName }
+
+// Plan implements BalanceStrategy.
+func (s *stickyBalanceStrategy) Plan(members map[string]ConsumerGroupMemberMetadata, topics map[string][]int32) (BalanceStrategyPlan, error) {
+ // track partition movements during generation of the partition assignment plan
+ s.movements = partitionMovements{
+ Movements: make(map[topicPartitionAssignment]consumerPair),
+ PartitionMovementsByTopic: make(map[string]map[consumerPair]map[topicPartitionAssignment]bool),
+ }
+
+ // prepopulate the current assignment state from userdata on the consumer group members
+ currentAssignment, prevAssignment, err := prepopulateCurrentAssignments(members)
+ if err != nil {
+ return nil, err
+ }
+
+ // determine if we're dealing with a completely fresh assignment, or if there's existing assignment state
+ isFreshAssignment := len(currentAssignment) == 0
+
+ // create a mapping of all current topic partitions and the consumers that can be assigned to them
+ partition2AllPotentialConsumers := make(map[topicPartitionAssignment][]string)
+ for topic, partitions := range topics {
+ for _, partition := range partitions {
+ partition2AllPotentialConsumers[topicPartitionAssignment{Topic: topic, Partition: partition}] = []string{}
+ }
+ }
+
+ // create a mapping of all consumers to all potential topic partitions that can be assigned to them
+ // also, populate the mapping of partitions to potential consumers
+ consumer2AllPotentialPartitions := make(map[string][]topicPartitionAssignment, len(members))
+ for memberID, meta := range members {
+ consumer2AllPotentialPartitions[memberID] = make([]topicPartitionAssignment, 0)
+ for _, topicSubscription := range meta.Topics {
+ // only evaluate topic subscriptions that are present in the supplied topics map
+ if _, found := topics[topicSubscription]; found {
+ for _, partition := range topics[topicSubscription] {
+ topicPartition := topicPartitionAssignment{Topic: topicSubscription, Partition: partition}
+ consumer2AllPotentialPartitions[memberID] = append(consumer2AllPotentialPartitions[memberID], topicPartition)
+ partition2AllPotentialConsumers[topicPartition] = append(partition2AllPotentialConsumers[topicPartition], memberID)
+ }
+ }
+ }
+
+ // add this consumer to currentAssignment (with an empty topic partition assignment) if it does not already exist
+ if _, exists := currentAssignment[memberID]; !exists {
+ currentAssignment[memberID] = make([]topicPartitionAssignment, 0)
+ }
+ }
+
+ // create a mapping of each partition to its current consumer, where possible
+ currentPartitionConsumers := make(map[topicPartitionAssignment]string, len(currentAssignment))
+ unvisitedPartitions := make(map[topicPartitionAssignment]bool, len(partition2AllPotentialConsumers))
+ for partition := range partition2AllPotentialConsumers {
+ unvisitedPartitions[partition] = true
+ }
+ var unassignedPartitions []topicPartitionAssignment
+ for memberID, partitions := range currentAssignment {
+ var keepPartitions []topicPartitionAssignment
+ for _, partition := range partitions {
+ // If this partition no longer exists at all, likely due to the
+ // topic being deleted, we remove the partition from the member.
+ if _, exists := partition2AllPotentialConsumers[partition]; !exists {
+ continue
+ }
+ delete(unvisitedPartitions, partition)
+ currentPartitionConsumers[partition] = memberID
+
+ if !slices.Contains(members[memberID].Topics, partition.Topic) {
+ unassignedPartitions = append(unassignedPartitions, partition)
+ continue
+ }
+ keepPartitions = append(keepPartitions, partition)
+ }
+ currentAssignment[memberID] = keepPartitions
+ }
+ for unvisited := range unvisitedPartitions {
+ unassignedPartitions = append(unassignedPartitions, unvisited)
+ }
+
+ // sort the topic partitions in order of priority for reassignment
+ sortedPartitions := sortPartitions(currentAssignment, prevAssignment, isFreshAssignment, partition2AllPotentialConsumers, consumer2AllPotentialPartitions)
+
+ // at this point we have preserved all valid topic partition to consumer assignments and removed
+ // all invalid topic partitions and invalid consumers. Now we need to assign unassignedPartitions
+ // to consumers so that the topic partition assignments are as balanced as possible.
+
+ // an ascending sorted set of consumers based on how many topic partitions are already assigned to them
+ sortedCurrentSubscriptions := sortMemberIDsByPartitionAssignments(currentAssignment)
+ s.balance(currentAssignment, prevAssignment, sortedPartitions, unassignedPartitions, sortedCurrentSubscriptions, consumer2AllPotentialPartitions, partition2AllPotentialConsumers, currentPartitionConsumers)
+
+ // Assemble plan
+ plan := make(BalanceStrategyPlan, len(currentAssignment))
+ for memberID, assignments := range currentAssignment {
+ if len(assignments) == 0 {
+ plan[memberID] = make(map[string][]int32)
+ } else {
+ for _, assignment := range assignments {
+ plan.Add(memberID, assignment.Topic, assignment.Partition)
+ }
+ }
+ }
+ return plan, nil
+}
+
+// AssignmentData serializes the set of topics currently assigned to the
+// specified member as part of the supplied balance plan
+func (s *stickyBalanceStrategy) AssignmentData(memberID string, topics map[string][]int32, generationID int32) ([]byte, error) {
+ return encode(&StickyAssignorUserDataV1{
+ Topics: topics,
+ Generation: generationID,
+ }, nil)
+}
+
+// Balance assignments across consumers for maximum fairness and stickiness.
+func (s *stickyBalanceStrategy) balance(currentAssignment map[string][]topicPartitionAssignment, prevAssignment map[topicPartitionAssignment]consumerGenerationPair, sortedPartitions []topicPartitionAssignment, unassignedPartitions []topicPartitionAssignment, sortedCurrentSubscriptions []string, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment, partition2AllPotentialConsumers map[topicPartitionAssignment][]string, currentPartitionConsumer map[topicPartitionAssignment]string) {
+ initializing := len(sortedCurrentSubscriptions) == 0 || len(currentAssignment[sortedCurrentSubscriptions[0]]) == 0
+
+ // assign all unassigned partitions
+ for _, partition := range unassignedPartitions {
+ // skip if there is no potential consumer for the partition
+ if len(partition2AllPotentialConsumers[partition]) == 0 {
+ continue
+ }
+ sortedCurrentSubscriptions = assignPartition(partition, sortedCurrentSubscriptions, currentAssignment, consumer2AllPotentialPartitions, currentPartitionConsumer)
+ }
+
+ // narrow down the reassignment scope to only those partitions that can actually be reassigned
+ for partition := range partition2AllPotentialConsumers {
+ if !canTopicPartitionParticipateInReassignment(partition, partition2AllPotentialConsumers) {
+ sortedPartitions = removeTopicPartitionFromMemberAssignments(sortedPartitions, partition)
+ }
+ }
+
+ // narrow down the reassignment scope to only those consumers that are subject to reassignment
+ fixedAssignments := make(map[string][]topicPartitionAssignment)
+ for memberID := range consumer2AllPotentialPartitions {
+ if !canConsumerParticipateInReassignment(memberID, currentAssignment, consumer2AllPotentialPartitions, partition2AllPotentialConsumers) {
+ fixedAssignments[memberID] = currentAssignment[memberID]
+ delete(currentAssignment, memberID)
+ sortedCurrentSubscriptions = sortMemberIDsByPartitionAssignments(currentAssignment)
+ }
+ }
+
+ // create a deep copy of the current assignment so we can revert to it if we do not get a more balanced assignment later
+ preBalanceAssignment := deepCopyAssignment(currentAssignment)
+ preBalancePartitionConsumers := maps.Clone(currentPartitionConsumer)
+
+ reassignmentPerformed := s.performReassignments(sortedPartitions, currentAssignment, prevAssignment, sortedCurrentSubscriptions, consumer2AllPotentialPartitions, partition2AllPotentialConsumers, currentPartitionConsumer)
+
+ // if we are not preserving existing assignments and we have made changes to the current assignment
+ // make sure we are getting a more balanced assignment; otherwise, revert to previous assignment
+ if !initializing && reassignmentPerformed && getBalanceScore(currentAssignment) >= getBalanceScore(preBalanceAssignment) {
+ currentAssignment = deepCopyAssignment(preBalanceAssignment)
+ clear(currentPartitionConsumer)
+ maps.Copy(currentPartitionConsumer, preBalancePartitionConsumers)
+ }
+
+ // add the fixed assignments (those that could not change) back
+ maps.Copy(currentAssignment, fixedAssignments)
+}
+
+// NewBalanceStrategyRoundRobin returns a round-robin balance strategy,
+// which assigns partitions to members in alternating order.
+// For example, there are two topics (t0, t1) and two consumer (m0, m1), and each topic has three partitions (p0, p1, p2):
+// M0: [t0p0, t0p2, t1p1]
+// M1: [t0p1, t1p0, t1p2]
+func NewBalanceStrategyRoundRobin() BalanceStrategy {
+ return new(roundRobinBalancer)
+}
+
+// Deprecated: use NewBalanceStrategyRoundRobin to avoid data race issue
+var BalanceStrategyRoundRobin = NewBalanceStrategyRoundRobin()
+
+type roundRobinBalancer struct{}
+
+func (b *roundRobinBalancer) Name() string {
+ return RoundRobinBalanceStrategyName
+}
+
+func (b *roundRobinBalancer) Plan(memberAndMetadata map[string]ConsumerGroupMemberMetadata, topics map[string][]int32) (BalanceStrategyPlan, error) {
+ if len(memberAndMetadata) == 0 || len(topics) == 0 {
+ return nil, errors.New("members and topics are not provided")
+ }
+ // sort partitions
+ var topicPartitions []topicAndPartition
+ for topic, partitions := range topics {
+ for _, partition := range partitions {
+ topicPartitions = append(topicPartitions, topicAndPartition{topic: topic, partition: partition})
+ }
+ }
+ sort.SliceStable(topicPartitions, func(i, j int) bool {
+ pi := topicPartitions[i]
+ pj := topicPartitions[j]
+ return pi.comparedValue() < pj.comparedValue()
+ })
+
+ // sort members
+ var members []memberAndTopic
+ for memberID, meta := range memberAndMetadata {
+ m := memberAndTopic{
+ memberID: memberID,
+ topics: make(map[string]struct{}),
+ }
+ for _, t := range meta.Topics {
+ m.topics[t] = struct{}{}
+ }
+ members = append(members, m)
+ }
+ sort.SliceStable(members, func(i, j int) bool {
+ mi := members[i]
+ mj := members[j]
+ return mi.memberID < mj.memberID
+ })
+
+ // assign partitions
+ plan := make(BalanceStrategyPlan, len(members))
+ i := 0
+ n := len(members)
+ for _, tp := range topicPartitions {
+ m := members[i%n]
+ for !m.hasTopic(tp.topic) {
+ i++
+ m = members[i%n]
+ }
+ plan.Add(m.memberID, tp.topic, tp.partition)
+ i++
+ }
+ return plan, nil
+}
+
+func (b *roundRobinBalancer) AssignmentData(memberID string, topics map[string][]int32, generationID int32) ([]byte, error) {
+ return nil, nil // do nothing for now
+}
+
+type topicAndPartition struct {
+ topic string
+ partition int32
+}
+
+func (tp *topicAndPartition) comparedValue() string {
+ return fmt.Sprintf("%s-%d", tp.topic, tp.partition)
+}
+
+type memberAndTopic struct {
+ topics map[string]struct{}
+ memberID string
+}
+
+func (m *memberAndTopic) hasTopic(topic string) bool {
+ _, isExist := m.topics[topic]
+ return isExist
+}
+
+// Calculate the balance score of the given assignment, as the sum of assigned partitions size difference of all consumer pairs.
+// A perfectly balanced assignment (with all consumers getting the same number of partitions) has a balance score of 0.
+// Lower balance score indicates a more balanced assignment.
+func getBalanceScore(assignment map[string][]topicPartitionAssignment) int {
+ consumer2AssignmentSize := make(map[string]int, len(assignment))
+ for memberID, partitions := range assignment {
+ consumer2AssignmentSize[memberID] = len(partitions)
+ }
+
+ var score float64
+ for memberID, consumerAssignmentSize := range consumer2AssignmentSize {
+ delete(consumer2AssignmentSize, memberID)
+ for _, otherConsumerAssignmentSize := range consumer2AssignmentSize {
+ score += math.Abs(float64(consumerAssignmentSize - otherConsumerAssignmentSize))
+ }
+ }
+ return int(score)
+}
+
+// Determine whether the current assignment plan is balanced.
+func isBalanced(currentAssignment map[string][]topicPartitionAssignment, allSubscriptions map[string][]topicPartitionAssignment) bool {
+ sortedCurrentSubscriptions := sortMemberIDsByPartitionAssignments(currentAssignment)
+ min := len(currentAssignment[sortedCurrentSubscriptions[0]])
+ max := len(currentAssignment[sortedCurrentSubscriptions[len(sortedCurrentSubscriptions)-1]])
+ if min >= max-1 {
+ // if minimum and maximum numbers of partitions assigned to consumers differ by at most one return true
+ return true
+ }
+
+ // create a mapping from partitions to the consumer assigned to them
+ allPartitions := make(map[topicPartitionAssignment]string)
+ for memberID, partitions := range currentAssignment {
+ for _, partition := range partitions {
+ if _, exists := allPartitions[partition]; exists {
+ Logger.Printf("Topic %s Partition %d is assigned more than one consumer", partition.Topic, partition.Partition)
+ }
+ allPartitions[partition] = memberID
+ }
+ }
+
+ // for each consumer that does not have all the topic partitions it can get make sure none of the topic partitions it
+ // could but did not get cannot be moved to it (because that would break the balance)
+ for _, memberID := range sortedCurrentSubscriptions {
+ consumerPartitions := currentAssignment[memberID]
+ consumerPartitionCount := len(consumerPartitions)
+
+ // skip if this consumer already has all the topic partitions it can get
+ if consumerPartitionCount == len(allSubscriptions[memberID]) {
+ continue
+ }
+
+ // otherwise make sure it cannot get any more
+ potentialTopicPartitions := allSubscriptions[memberID]
+ for _, partition := range potentialTopicPartitions {
+ if !memberAssignmentsIncludeTopicPartition(currentAssignment[memberID], partition) {
+ otherConsumer := allPartitions[partition]
+ otherConsumerPartitionCount := len(currentAssignment[otherConsumer])
+ if consumerPartitionCount < otherConsumerPartitionCount {
+ return false
+ }
+ }
+ }
+ }
+ return true
+}
+
+// Reassign all topic partitions that need reassignment until balanced.
+func (s *stickyBalanceStrategy) performReassignments(reassignablePartitions []topicPartitionAssignment, currentAssignment map[string][]topicPartitionAssignment, prevAssignment map[topicPartitionAssignment]consumerGenerationPair, sortedCurrentSubscriptions []string, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment, partition2AllPotentialConsumers map[topicPartitionAssignment][]string, currentPartitionConsumer map[topicPartitionAssignment]string) bool {
+ reassignmentPerformed := false
+ modified := false
+
+ // repeat reassignment until no partition can be moved to improve the balance
+ for {
+ modified = false
+ // reassign all reassignable partitions (starting from the partition with least potential consumers and if needed)
+ // until the full list is processed or a balance is achieved
+ for _, partition := range reassignablePartitions {
+ if isBalanced(currentAssignment, consumer2AllPotentialPartitions) {
+ break
+ }
+
+ // the partition must have at least two consumers
+ if len(partition2AllPotentialConsumers[partition]) <= 1 {
+ Logger.Printf("Expected more than one potential consumer for partition %s topic %d", partition.Topic, partition.Partition)
+ }
+
+ // the partition must have a consumer
+ consumer := currentPartitionConsumer[partition]
+ if consumer == "" {
+ Logger.Printf("Expected topic %s partition %d to be assigned to a consumer", partition.Topic, partition.Partition)
+ }
+
+ if _, exists := prevAssignment[partition]; exists {
+ if len(currentAssignment[consumer]) > (len(currentAssignment[prevAssignment[partition].MemberID]) + 1) {
+ sortedCurrentSubscriptions = s.reassignPartition(partition, currentAssignment, sortedCurrentSubscriptions, currentPartitionConsumer, prevAssignment[partition].MemberID)
+ reassignmentPerformed = true
+ modified = true
+ continue
+ }
+ }
+
+ // check if a better-suited consumer exists for the partition; if so, reassign it
+ for _, otherConsumer := range partition2AllPotentialConsumers[partition] {
+ if len(currentAssignment[consumer]) > (len(currentAssignment[otherConsumer]) + 1) {
+ sortedCurrentSubscriptions = s.reassignPartitionToNewConsumer(partition, currentAssignment, sortedCurrentSubscriptions, currentPartitionConsumer, consumer2AllPotentialPartitions)
+ reassignmentPerformed = true
+ modified = true
+ break
+ }
+ }
+ }
+ if !modified {
+ return reassignmentPerformed
+ }
+ }
+}
+
+// Identify a new consumer for a topic partition and reassign it.
+func (s *stickyBalanceStrategy) reassignPartitionToNewConsumer(partition topicPartitionAssignment, currentAssignment map[string][]topicPartitionAssignment, sortedCurrentSubscriptions []string, currentPartitionConsumer map[topicPartitionAssignment]string, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment) []string {
+ for _, anotherConsumer := range sortedCurrentSubscriptions {
+ if memberAssignmentsIncludeTopicPartition(consumer2AllPotentialPartitions[anotherConsumer], partition) {
+ return s.reassignPartition(partition, currentAssignment, sortedCurrentSubscriptions, currentPartitionConsumer, anotherConsumer)
+ }
+ }
+ return sortedCurrentSubscriptions
+}
+
+// Reassign a specific partition to a new consumer
+func (s *stickyBalanceStrategy) reassignPartition(partition topicPartitionAssignment, currentAssignment map[string][]topicPartitionAssignment, sortedCurrentSubscriptions []string, currentPartitionConsumer map[topicPartitionAssignment]string, newConsumer string) []string {
+ consumer := currentPartitionConsumer[partition]
+ // find the correct partition movement considering the stickiness requirement
+ partitionToBeMoved := s.movements.getTheActualPartitionToBeMoved(partition, consumer, newConsumer)
+ return s.processPartitionMovement(partitionToBeMoved, newConsumer, currentAssignment, sortedCurrentSubscriptions, currentPartitionConsumer)
+}
+
+// Track the movement of a topic partition after assignment
+func (s *stickyBalanceStrategy) processPartitionMovement(partition topicPartitionAssignment, newConsumer string, currentAssignment map[string][]topicPartitionAssignment, sortedCurrentSubscriptions []string, currentPartitionConsumer map[topicPartitionAssignment]string) []string {
+ oldConsumer := currentPartitionConsumer[partition]
+ s.movements.movePartition(partition, oldConsumer, newConsumer)
+
+ currentAssignment[oldConsumer] = removeTopicPartitionFromMemberAssignments(currentAssignment[oldConsumer], partition)
+ currentAssignment[newConsumer] = append(currentAssignment[newConsumer], partition)
+ currentPartitionConsumer[partition] = newConsumer
+ return sortMemberIDsByPartitionAssignments(currentAssignment)
+}
+
+// Determine whether a specific consumer should be considered for topic partition assignment.
+func canConsumerParticipateInReassignment(memberID string, currentAssignment map[string][]topicPartitionAssignment, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment, partition2AllPotentialConsumers map[topicPartitionAssignment][]string) bool {
+ currentPartitions := currentAssignment[memberID]
+ currentAssignmentSize := len(currentPartitions)
+ maxAssignmentSize := len(consumer2AllPotentialPartitions[memberID])
+ if currentAssignmentSize > maxAssignmentSize {
+ Logger.Printf("The consumer %s is assigned more partitions than the maximum possible", memberID)
+ }
+ if currentAssignmentSize < maxAssignmentSize {
+ // if a consumer is not assigned all its potential partitions it is subject to reassignment
+ return true
+ }
+ for _, partition := range currentPartitions {
+ if canTopicPartitionParticipateInReassignment(partition, partition2AllPotentialConsumers) {
+ return true
+ }
+ }
+ return false
+}
+
+// Only consider reassigning those topic partitions that have two or more potential consumers.
+func canTopicPartitionParticipateInReassignment(partition topicPartitionAssignment, partition2AllPotentialConsumers map[topicPartitionAssignment][]string) bool {
+ return len(partition2AllPotentialConsumers[partition]) >= 2
+}
+
+// The assignment should improve the overall balance of the partition assignments to consumers.
+func assignPartition(partition topicPartitionAssignment, sortedCurrentSubscriptions []string, currentAssignment map[string][]topicPartitionAssignment, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment, currentPartitionConsumer map[topicPartitionAssignment]string) []string {
+ for _, memberID := range sortedCurrentSubscriptions {
+ if memberAssignmentsIncludeTopicPartition(consumer2AllPotentialPartitions[memberID], partition) {
+ currentAssignment[memberID] = append(currentAssignment[memberID], partition)
+ currentPartitionConsumer[partition] = memberID
+ break
+ }
+ }
+ return sortMemberIDsByPartitionAssignments(currentAssignment)
+}
+
+// Deserialize topic partition assignment data to aid with creation of a sticky assignment.
+func deserializeTopicPartitionAssignment(userDataBytes []byte) (StickyAssignorUserData, error) {
+ userDataV1 := &StickyAssignorUserDataV1{}
+ if err := decode(userDataBytes, userDataV1, nil); err != nil {
+ userDataV0 := &StickyAssignorUserDataV0{}
+ if err := decode(userDataBytes, userDataV0, nil); err != nil {
+ return nil, err
+ }
+ return userDataV0, nil
+ }
+ return userDataV1, nil
+}
+
+// filterAssignedPartitions returns a map of consumer group members to their list of previously-assigned topic partitions, limited
+// to those topic partitions currently reported by the Kafka cluster.
+func filterAssignedPartitions(currentAssignment map[string][]topicPartitionAssignment, partition2AllPotentialConsumers map[topicPartitionAssignment][]string) map[string][]topicPartitionAssignment {
+ assignments := deepCopyAssignment(currentAssignment)
+ for memberID, partitions := range assignments {
+ // perform in-place filtering
+ i := 0
+ for _, partition := range partitions {
+ if _, exists := partition2AllPotentialConsumers[partition]; exists {
+ partitions[i] = partition
+ i++
+ }
+ }
+ assignments[memberID] = partitions[:i]
+ }
+ return assignments
+}
+
+func removeTopicPartitionFromMemberAssignments(assignments []topicPartitionAssignment, topic topicPartitionAssignment) []topicPartitionAssignment {
+ for i, assignment := range assignments {
+ if assignment == topic {
+ return append(assignments[:i], assignments[i+1:]...)
+ }
+ }
+ return assignments
+}
+
+func memberAssignmentsIncludeTopicPartition(assignments []topicPartitionAssignment, topic topicPartitionAssignment) bool {
+ return slices.Contains(assignments, topic)
+}
+
+func sortPartitions(currentAssignment map[string][]topicPartitionAssignment, partitionsWithADifferentPreviousAssignment map[topicPartitionAssignment]consumerGenerationPair, isFreshAssignment bool, partition2AllPotentialConsumers map[topicPartitionAssignment][]string, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment) []topicPartitionAssignment {
+ unassignedPartitions := make(map[topicPartitionAssignment]bool, len(partition2AllPotentialConsumers))
+ for partition := range partition2AllPotentialConsumers {
+ unassignedPartitions[partition] = true
+ }
+
+ sortedPartitions := make([]topicPartitionAssignment, 0)
+ if !isFreshAssignment && areSubscriptionsIdentical(partition2AllPotentialConsumers, consumer2AllPotentialPartitions) {
+ // if this is a reassignment and the subscriptions are identical (all consumers can consumer from all topics)
+ // then we just need to simply list partitions in a round robin fashion (from consumers with
+ // most assigned partitions to those with least)
+ assignments := filterAssignedPartitions(currentAssignment, partition2AllPotentialConsumers)
+
+ // use priority-queue to evaluate consumer group members in descending-order based on
+ // the number of topic partition assignments (i.e. consumers with most assignments first)
+ pq := make(assignmentPriorityQueue, len(assignments))
+ i := 0
+ for consumerID, consumerAssignments := range assignments {
+ pq[i] = &consumerGroupMember{
+ id: consumerID,
+ assignments: consumerAssignments,
+ }
+ i++
+ }
+ heap.Init(&pq)
+
+ // loop until no consumer-group members remain
+ for pq.Len() != 0 {
+ member := pq[0]
+
+ // partitions that were assigned to a different consumer last time
+ var prevPartitionIndex int
+ for i, partition := range member.assignments {
+ if _, exists := partitionsWithADifferentPreviousAssignment[partition]; exists {
+ prevPartitionIndex = i
+ break
+ }
+ }
+
+ if len(member.assignments) > 0 {
+ partition := member.assignments[prevPartitionIndex]
+ sortedPartitions = append(sortedPartitions, partition)
+ delete(unassignedPartitions, partition)
+ if prevPartitionIndex == 0 {
+ member.assignments = member.assignments[1:]
+ } else {
+ member.assignments = append(member.assignments[:prevPartitionIndex], member.assignments[prevPartitionIndex+1:]...)
+ }
+ heap.Fix(&pq, 0)
+ } else {
+ heap.Pop(&pq)
+ }
+ }
+
+ for partition := range unassignedPartitions {
+ sortedPartitions = append(sortedPartitions, partition)
+ }
+ } else {
+ // an ascending sorted set of topic partitions based on how many consumers can potentially use them
+ sortedPartitions = sortPartitionsByPotentialConsumerAssignments(partition2AllPotentialConsumers)
+ }
+ return sortedPartitions
+}
+
+func sortMemberIDsByPartitionAssignments(assignments map[string][]topicPartitionAssignment) []string {
+ // sort the members by the number of partition assignments in ascending order
+ sortedMemberIDs := make([]string, 0, len(assignments))
+ for memberID := range assignments {
+ sortedMemberIDs = append(sortedMemberIDs, memberID)
+ }
+ sort.SliceStable(sortedMemberIDs, func(i, j int) bool {
+ ret := len(assignments[sortedMemberIDs[i]]) - len(assignments[sortedMemberIDs[j]])
+ if ret == 0 {
+ return sortedMemberIDs[i] < sortedMemberIDs[j]
+ }
+ return len(assignments[sortedMemberIDs[i]]) < len(assignments[sortedMemberIDs[j]])
+ })
+ return sortedMemberIDs
+}
+
+func sortPartitionsByPotentialConsumerAssignments(partition2AllPotentialConsumers map[topicPartitionAssignment][]string) []topicPartitionAssignment {
+ // sort the members by the number of partition assignments in descending order
+ sortedPartionIDs := make([]topicPartitionAssignment, len(partition2AllPotentialConsumers))
+ i := 0
+ for partition := range partition2AllPotentialConsumers {
+ sortedPartionIDs[i] = partition
+ i++
+ }
+ sort.Slice(sortedPartionIDs, func(i, j int) bool {
+ if len(partition2AllPotentialConsumers[sortedPartionIDs[i]]) == len(partition2AllPotentialConsumers[sortedPartionIDs[j]]) {
+ ret := strings.Compare(sortedPartionIDs[i].Topic, sortedPartionIDs[j].Topic)
+ if ret == 0 {
+ return sortedPartionIDs[i].Partition < sortedPartionIDs[j].Partition
+ }
+ return ret < 0
+ }
+ return len(partition2AllPotentialConsumers[sortedPartionIDs[i]]) < len(partition2AllPotentialConsumers[sortedPartionIDs[j]])
+ })
+ return sortedPartionIDs
+}
+
+func deepCopyAssignment(assignment map[string][]topicPartitionAssignment) map[string][]topicPartitionAssignment {
+ m := make(map[string][]topicPartitionAssignment, len(assignment))
+ for memberID, subscriptions := range assignment {
+ m[memberID] = append(subscriptions[:0:0], subscriptions...)
+ }
+ return m
+}
+
+func areSubscriptionsIdentical(partition2AllPotentialConsumers map[topicPartitionAssignment][]string, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment) bool {
+ curMembers := make(map[string]int)
+ for _, cur := range partition2AllPotentialConsumers {
+ if len(curMembers) == 0 {
+ for _, curMembersElem := range cur {
+ curMembers[curMembersElem]++
+ }
+ continue
+ }
+
+ if len(curMembers) != len(cur) {
+ return false
+ }
+
+ yMap := make(map[string]int)
+ for _, yElem := range cur {
+ yMap[yElem]++
+ }
+
+ for curMembersMapKey, curMembersMapVal := range curMembers {
+ if yMap[curMembersMapKey] != curMembersMapVal {
+ return false
+ }
+ }
+ }
+
+ curPartitions := make(map[topicPartitionAssignment]int)
+ for _, cur := range consumer2AllPotentialPartitions {
+ if len(curPartitions) == 0 {
+ for _, curPartitionElem := range cur {
+ curPartitions[curPartitionElem]++
+ }
+ continue
+ }
+
+ if len(curPartitions) != len(cur) {
+ return false
+ }
+
+ yMap := make(map[topicPartitionAssignment]int)
+ for _, yElem := range cur {
+ yMap[yElem]++
+ }
+
+ for curMembersMapKey, curMembersMapVal := range curPartitions {
+ if yMap[curMembersMapKey] != curMembersMapVal {
+ return false
+ }
+ }
+ }
+ return true
+}
+
+// We need to process subscriptions' user data with each consumer's reported generation in mind
+// higher generations overwrite lower generations in case of a conflict
+// note that a conflict could exist only if user data is for different generations
+func prepopulateCurrentAssignments(members map[string]ConsumerGroupMemberMetadata) (map[string][]topicPartitionAssignment, map[topicPartitionAssignment]consumerGenerationPair, error) {
+ currentAssignment := make(map[string][]topicPartitionAssignment)
+ prevAssignment := make(map[topicPartitionAssignment]consumerGenerationPair)
+
+ // for each partition we create a sorted map of its consumers by generation
+ sortedPartitionConsumersByGeneration := make(map[topicPartitionAssignment]map[int]string)
+ for memberID, meta := range members {
+ consumerUserData, err := deserializeTopicPartitionAssignment(meta.UserData)
+ if err != nil {
+ return nil, nil, err
+ }
+ for _, partition := range consumerUserData.partitions() {
+ if consumers, exists := sortedPartitionConsumersByGeneration[partition]; exists {
+ if consumerUserData.hasGeneration() {
+ if _, generationExists := consumers[consumerUserData.generation()]; generationExists {
+ // same partition is assigned to two consumers during the same rebalance.
+ // log a warning and skip this record
+ Logger.Printf("Topic %s Partition %d is assigned to multiple consumers following sticky assignment generation %d", partition.Topic, partition.Partition, consumerUserData.generation())
+ continue
+ } else {
+ consumers[consumerUserData.generation()] = memberID
+ }
+ } else {
+ consumers[defaultGeneration] = memberID
+ }
+ } else {
+ generation := defaultGeneration
+ if consumerUserData.hasGeneration() {
+ generation = consumerUserData.generation()
+ }
+ sortedPartitionConsumersByGeneration[partition] = map[int]string{generation: memberID}
+ }
+ }
+ }
+
+ // prevAssignment holds the prior ConsumerGenerationPair (before current) of each partition
+ // current and previous consumers are the last two consumers of each partition in the above sorted map
+ for partition, consumers := range sortedPartitionConsumersByGeneration {
+ // sort consumers by generation in decreasing order
+ var generations []int
+ for generation := range consumers {
+ generations = append(generations, generation)
+ }
+ sort.Sort(sort.Reverse(sort.IntSlice(generations)))
+
+ consumer := consumers[generations[0]]
+ if _, exists := currentAssignment[consumer]; !exists {
+ currentAssignment[consumer] = []topicPartitionAssignment{partition}
+ } else {
+ currentAssignment[consumer] = append(currentAssignment[consumer], partition)
+ }
+
+ // check for previous assignment, if any
+ if len(generations) > 1 {
+ prevAssignment[partition] = consumerGenerationPair{
+ MemberID: consumers[generations[1]],
+ Generation: generations[1],
+ }
+ }
+ }
+ return currentAssignment, prevAssignment, nil
+}
+
+type consumerGenerationPair struct {
+ MemberID string
+ Generation int
+}
+
+// consumerPair represents a pair of Kafka consumer ids involved in a partition reassignment.
+type consumerPair struct {
+ SrcMemberID string
+ DstMemberID string
+}
+
+// partitionMovements maintains some data structures to simplify lookup of partition movements among consumers.
+type partitionMovements struct {
+ PartitionMovementsByTopic map[string]map[consumerPair]map[topicPartitionAssignment]bool
+ Movements map[topicPartitionAssignment]consumerPair
+}
+
+func (p *partitionMovements) removeMovementRecordOfPartition(partition topicPartitionAssignment) consumerPair {
+ pair := p.Movements[partition]
+ delete(p.Movements, partition)
+
+ partitionMovementsForThisTopic := p.PartitionMovementsByTopic[partition.Topic]
+ delete(partitionMovementsForThisTopic[pair], partition)
+ if len(partitionMovementsForThisTopic[pair]) == 0 {
+ delete(partitionMovementsForThisTopic, pair)
+ }
+ if len(p.PartitionMovementsByTopic[partition.Topic]) == 0 {
+ delete(p.PartitionMovementsByTopic, partition.Topic)
+ }
+ return pair
+}
+
+func (p *partitionMovements) addPartitionMovementRecord(partition topicPartitionAssignment, pair consumerPair) {
+ p.Movements[partition] = pair
+ if _, exists := p.PartitionMovementsByTopic[partition.Topic]; !exists {
+ p.PartitionMovementsByTopic[partition.Topic] = make(map[consumerPair]map[topicPartitionAssignment]bool)
+ }
+ partitionMovementsForThisTopic := p.PartitionMovementsByTopic[partition.Topic]
+ if _, exists := partitionMovementsForThisTopic[pair]; !exists {
+ partitionMovementsForThisTopic[pair] = make(map[topicPartitionAssignment]bool)
+ }
+ partitionMovementsForThisTopic[pair][partition] = true
+}
+
+func (p *partitionMovements) movePartition(partition topicPartitionAssignment, oldConsumer, newConsumer string) {
+ pair := consumerPair{
+ SrcMemberID: oldConsumer,
+ DstMemberID: newConsumer,
+ }
+ if _, exists := p.Movements[partition]; exists {
+ // this partition has previously moved
+ existingPair := p.removeMovementRecordOfPartition(partition)
+ if existingPair.DstMemberID != oldConsumer {
+ Logger.Printf("Existing pair DstMemberID %s was not equal to the oldConsumer ID %s", existingPair.DstMemberID, oldConsumer)
+ }
+ if existingPair.SrcMemberID != newConsumer {
+ // the partition is not moving back to its previous consumer
+ p.addPartitionMovementRecord(partition, consumerPair{
+ SrcMemberID: existingPair.SrcMemberID,
+ DstMemberID: newConsumer,
+ })
+ }
+ } else {
+ p.addPartitionMovementRecord(partition, pair)
+ }
+}
+
+func (p *partitionMovements) getTheActualPartitionToBeMoved(partition topicPartitionAssignment, oldConsumer, newConsumer string) topicPartitionAssignment {
+ if _, exists := p.PartitionMovementsByTopic[partition.Topic]; !exists {
+ return partition
+ }
+ if _, exists := p.Movements[partition]; exists {
+ // this partition has previously moved
+ if oldConsumer != p.Movements[partition].DstMemberID {
+ Logger.Printf("Partition movement DstMemberID %s was not equal to the oldConsumer ID %s", p.Movements[partition].DstMemberID, oldConsumer)
+ }
+ oldConsumer = p.Movements[partition].SrcMemberID
+ }
+
+ partitionMovementsForThisTopic := p.PartitionMovementsByTopic[partition.Topic]
+ reversePair := consumerPair{
+ SrcMemberID: newConsumer,
+ DstMemberID: oldConsumer,
+ }
+ if _, exists := partitionMovementsForThisTopic[reversePair]; !exists {
+ return partition
+ }
+ var reversePairPartition topicPartitionAssignment
+ for otherPartition := range partitionMovementsForThisTopic[reversePair] {
+ reversePairPartition = otherPartition
+ }
+ return reversePairPartition
+}
+
+//lint:ignore U1000 // this is used but only in unittests as a helper (which are excluded by the integration build tag)
+func (p *partitionMovements) isLinked(src, dst string, pairs []consumerPair, currentPath []string) ([]string, bool) {
+ if src == dst {
+ return currentPath, false
+ }
+ if len(pairs) == 0 {
+ return currentPath, false
+ }
+ for _, pair := range pairs {
+ if src == pair.SrcMemberID && dst == pair.DstMemberID {
+ currentPath = append(currentPath, src, dst)
+ return currentPath, true
+ }
+ }
+
+ for _, pair := range pairs {
+ if pair.SrcMemberID != src {
+ continue
+ }
+ // create a deep copy of the pairs, excluding the current pair
+ reducedSet := make([]consumerPair, len(pairs)-1)
+ i := 0
+ for _, p := range pairs {
+ if p != pair {
+ reducedSet[i] = pair
+ i++
+ }
+ }
+
+ currentPath = append(currentPath, pair.SrcMemberID)
+ return p.isLinked(pair.DstMemberID, dst, reducedSet, currentPath)
+ }
+ return currentPath, false
+}
+
+//lint:ignore U1000 // this is used but only in unittests as a helper (which are excluded by the integration build tag)
+func (p *partitionMovements) in(cycle []string, cycles [][]string) bool {
+ superCycle := make([]string, len(cycle)-1)
+ for i := 0; i < len(cycle)-1; i++ {
+ superCycle[i] = cycle[i]
+ }
+ superCycle = append(superCycle, cycle...)
+ for _, foundCycle := range cycles {
+ if len(foundCycle) == len(cycle) && indexOfSubList(superCycle, foundCycle) != -1 {
+ return true
+ }
+ }
+ return false
+}
+
+//lint:ignore U1000 // this is used but only in unittests as a helper (which are excluded by the integration build tag)
+func (p *partitionMovements) hasCycles(pairs []consumerPair) bool {
+ cycles := make([][]string, 0)
+ for _, pair := range pairs {
+ // create a deep copy of the pairs, excluding the current pair
+ reducedPairs := make([]consumerPair, len(pairs)-1)
+ i := 0
+ for _, p := range pairs {
+ if p != pair {
+ reducedPairs[i] = pair
+ i++
+ }
+ }
+ if path, linked := p.isLinked(pair.DstMemberID, pair.SrcMemberID, reducedPairs, []string{pair.SrcMemberID}); linked {
+ if !p.in(path, cycles) {
+ cycles = append(cycles, path)
+ Logger.Printf("A cycle of length %d was found: %v", len(path)-1, path)
+ }
+ }
+ }
+
+ // for now we want to make sure there is no partition movements of the same topic between a pair of consumers.
+ // the odds of finding a cycle among more than two consumers seem to be very low (according to various randomized
+ // tests with the given sticky algorithm) that it should not worth the added complexity of handling those cases.
+ for _, cycle := range cycles {
+ if len(cycle) == 3 {
+ return true
+ }
+ }
+ return false
+}
+
+//lint:ignore U1000 // this is used but only in unittests as a helper (which are excluded by the integration build tag)
+func (p *partitionMovements) isSticky() bool {
+ for topic, movements := range p.PartitionMovementsByTopic {
+ movementPairs := make([]consumerPair, len(movements))
+ i := 0
+ for pair := range movements {
+ movementPairs[i] = pair
+ i++
+ }
+ if p.hasCycles(movementPairs) {
+ Logger.Printf("Stickiness is violated for topic %s", topic)
+ Logger.Printf("Partition movements for this topic occurred among the following consumer pairs: %v", movements)
+ return false
+ }
+ }
+ return true
+}
+
+//lint:ignore U1000 // this is used but only in unittests as a helper (which are excluded by the integration build tag)
+func indexOfSubList(source []string, target []string) int {
+ targetSize := len(target)
+ maxCandidate := len(source) - targetSize
+nextCand:
+ for candidate := 0; candidate <= maxCandidate; candidate++ {
+ j := candidate
+ for i := 0; i < targetSize; i++ {
+ if target[i] != source[j] {
+ // Element mismatch, try next cand
+ continue nextCand
+ }
+ j++
+ }
+ // All elements of candidate matched target
+ return candidate
+ }
+ return -1
+}
+
+type consumerGroupMember struct {
+ id string
+ assignments []topicPartitionAssignment
+}
+
+// assignmentPriorityQueue is a priority-queue of consumer group members that is sorted
+// in descending order (most assignments to least assignments).
+type assignmentPriorityQueue []*consumerGroupMember
+
+func (pq assignmentPriorityQueue) Len() int { return len(pq) }
+
+func (pq assignmentPriorityQueue) Less(i, j int) bool {
+ // order assignment priority queue in descending order using assignment-count/member-id
+ if len(pq[i].assignments) == len(pq[j].assignments) {
+ return pq[i].id > pq[j].id
+ }
+ return len(pq[i].assignments) > len(pq[j].assignments)
+}
+
+func (pq assignmentPriorityQueue) Swap(i, j int) {
+ pq[i], pq[j] = pq[j], pq[i]
+}
+
+func (pq *assignmentPriorityQueue) Push(x interface{}) {
+ member := x.(*consumerGroupMember)
+ *pq = append(*pq, member)
+}
+
+func (pq *assignmentPriorityQueue) Pop() interface{} {
+ old := *pq
+ n := len(old)
+ member := old[n-1]
+ *pq = old[0 : n-1]
+ return member
+}
diff --git a/vendor/github.com/IBM/sarama/broker.go b/vendor/github.com/IBM/sarama/broker.go
new file mode 100644
index 0000000..7c559cf
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/broker.go
@@ -0,0 +1,1869 @@
+package sarama
+
+import (
+ "crypto/tls"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "io"
+ "math/rand"
+ "net"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/rcrowley/go-metrics"
+)
+
+// Broker represents a single Kafka broker connection. All operations on this object are entirely concurrency-safe.
+type Broker struct {
+ conf *Config
+ rack *string
+
+ id int32
+ addr string
+ correlationID int32
+ conn net.Conn
+ connErr error
+ lock sync.Mutex
+ opened atomic.Bool
+ responses chan *responsePromise
+ done chan bool
+
+ metricRegistry metrics.Registry
+ incomingByteRate metrics.Meter
+ requestRate metrics.Meter
+ fetchRate metrics.Meter
+ requestSize metrics.Histogram
+ requestLatency metrics.Histogram
+ outgoingByteRate metrics.Meter
+ responseRate metrics.Meter
+ responseSize metrics.Histogram
+ requestsInFlight metrics.Counter
+ protocolRequestsRate map[int16]metrics.Meter
+ brokerIncomingByteRate metrics.Meter
+ brokerRequestRate metrics.Meter
+ brokerFetchRate metrics.Meter
+ brokerRequestSize metrics.Histogram
+ brokerRequestLatency metrics.Histogram
+ brokerOutgoingByteRate metrics.Meter
+ brokerResponseRate metrics.Meter
+ brokerResponseSize metrics.Histogram
+ brokerRequestsInFlight metrics.Counter
+ brokerThrottleTime metrics.Histogram
+ brokerProtocolRequestsRate map[int16]metrics.Meter
+ brokerAPIVersions apiVersionMap
+
+ kerberosAuthenticator GSSAPIKerberosAuth
+ clientSessionReauthenticationTimeMs int64
+
+ throttleTimer *time.Timer
+ throttleTimerLock sync.Mutex
+}
+
+// SASLMechanism specifies the SASL mechanism the client uses to authenticate with the broker
+type SASLMechanism string
+
+const (
+ // SASLTypeOAuth represents the SASL/OAUTHBEARER mechanism (Kafka 2.0.0+)
+ SASLTypeOAuth = "OAUTHBEARER"
+ // SASLTypePlaintext represents the SASL/PLAIN mechanism
+ SASLTypePlaintext = "PLAIN"
+ // SASLTypeSCRAMSHA256 represents the SCRAM-SHA-256 mechanism.
+ SASLTypeSCRAMSHA256 = "SCRAM-SHA-256"
+ // SASLTypeSCRAMSHA512 represents the SCRAM-SHA-512 mechanism.
+ SASLTypeSCRAMSHA512 = "SCRAM-SHA-512"
+ SASLTypeGSSAPI = "GSSAPI"
+ // SASLHandshakeV0 is v0 of the Kafka SASL handshake protocol. Client and
+ // server negotiate SASL auth using opaque packets.
+ SASLHandshakeV0 = int16(0)
+ // SASLHandshakeV1 is v1 of the Kafka SASL handshake protocol. Client and
+ // server negotiate SASL by wrapping tokens with Kafka protocol headers.
+ SASLHandshakeV1 = int16(1)
+ // SASLExtKeyAuth is the reserved extension key name sent as part of the
+ // SASL/OAUTHBEARER initial client response
+ SASLExtKeyAuth = "auth"
+)
+
+// AccessToken contains an access token used to authenticate a
+// SASL/OAUTHBEARER client along with associated metadata.
+type AccessToken struct {
+ // Token is the access token payload.
+ Token string
+ // Extensions is a optional map of arbitrary key-value pairs that can be
+ // sent with the SASL/OAUTHBEARER initial client response. These values are
+ // ignored by the SASL server if they are unexpected. This feature is only
+ // supported by Kafka >= 2.1.0.
+ Extensions map[string]string
+}
+
+// AccessTokenProvider is the interface that encapsulates how implementors
+// can generate access tokens for Kafka broker authentication.
+type AccessTokenProvider interface {
+ // Token returns an access token. The implementation should ensure token
+ // reuse so that multiple calls at connect time do not create multiple
+ // tokens. The implementation should also periodically refresh the token in
+ // order to guarantee that each call returns an unexpired token. This
+ // method should not block indefinitely--a timeout error should be returned
+ // after a short period of inactivity so that the broker connection logic
+ // can log debugging information and retry.
+ Token() (*AccessToken, error)
+}
+
+// SCRAMClient is a an interface to a SCRAM
+// client implementation.
+type SCRAMClient interface {
+ // Begin prepares the client for the SCRAM exchange
+ // with the server with a user name and a password
+ Begin(userName, password, authzID string) error
+ // Step steps client through the SCRAM exchange. It is
+ // called repeatedly until it errors or `Done` returns true.
+ Step(challenge string) (response string, err error)
+ // Done should return true when the SCRAM conversation
+ // is over.
+ Done() bool
+}
+
+type responsePromise struct {
+ requestTime time.Time
+ correlationID int32
+ response protocolBody
+ handler func([]byte, error)
+ packets chan []byte
+ errors chan error
+}
+
+func (p *responsePromise) handle(packets []byte, err error) {
+ // Use callback when provided
+ if p.handler != nil {
+ p.handler(packets, err)
+ return
+ }
+ // Otherwise fallback to using channels
+ if err != nil {
+ p.errors <- err
+ return
+ }
+ p.packets <- packets
+}
+
+// NewBroker creates and returns a Broker targeting the given host:port address.
+// This does not attempt to actually connect, you have to call Open() for that.
+func NewBroker(addr string) *Broker {
+ return &Broker{id: -1, addr: addr}
+}
+
+// Open tries to connect to the Broker if it is not already connected or connecting, but does not block
+// waiting for the connection to complete. This means that any subsequent operations on the broker will
+// block waiting for the connection to succeed or fail. To get the effect of a fully synchronous Open call,
+// follow it by a call to Connected(). The only errors Open will return directly are ConfigurationError or
+// AlreadyConnected. If conf is nil, the result of NewConfig() is used.
+func (b *Broker) Open(conf *Config) error {
+ if !b.opened.CompareAndSwap(false, true) {
+ return ErrAlreadyConnected
+ }
+
+ if conf == nil {
+ conf = NewConfig()
+ }
+
+ err := conf.Validate()
+ if err != nil {
+ return err
+ }
+
+ b.lock.Lock()
+
+ if b.metricRegistry == nil {
+ b.metricRegistry = newCleanupRegistry(conf.MetricRegistry)
+ }
+
+ go withRecover(func() {
+ defer b.lock.Unlock()
+
+ dialer := conf.getDialer()
+ b.conn, b.connErr = dialer.Dial("tcp", b.addr)
+ if b.connErr != nil {
+ Logger.Printf("Failed to connect to broker %s: %s\n", b.addr, b.connErr)
+ b.conn = nil
+ b.opened.Store(false)
+ return
+ }
+ if conf.Net.TLS.Enable {
+ b.conn = tls.Client(b.conn, validServerNameTLS(b.addr, conf.Net.TLS.Config))
+ }
+
+ b.conn = newBufConn(b.conn)
+ b.conf = conf
+
+ // Create or reuse the global metrics shared between brokers
+ b.incomingByteRate = metrics.GetOrRegisterMeter("incoming-byte-rate", b.metricRegistry)
+ b.requestRate = metrics.GetOrRegisterMeter("request-rate", b.metricRegistry)
+ b.fetchRate = metrics.GetOrRegisterMeter("consumer-fetch-rate", b.metricRegistry)
+ b.requestSize = getOrRegisterHistogram("request-size", b.metricRegistry)
+ b.requestLatency = getOrRegisterHistogram("request-latency-in-ms", b.metricRegistry)
+ b.outgoingByteRate = metrics.GetOrRegisterMeter("outgoing-byte-rate", b.metricRegistry)
+ b.responseRate = metrics.GetOrRegisterMeter("response-rate", b.metricRegistry)
+ b.responseSize = getOrRegisterHistogram("response-size", b.metricRegistry)
+ b.requestsInFlight = metrics.GetOrRegisterCounter("requests-in-flight", b.metricRegistry)
+ b.protocolRequestsRate = map[int16]metrics.Meter{}
+ // Do not gather metrics for seeded broker (only used during bootstrap) because they share
+ // the same id (-1) and are already exposed through the global metrics above
+ if b.id >= 0 && !metrics.UseNilMetrics {
+ b.registerMetrics()
+ }
+
+ // Send an ApiVersionsRequest to identify the client (KIP-511).
+ // Store the response in the brokerAPIVersions map.
+ // It will be used to determine the supported API versions for each request.
+ // This should happen before SASL authentication: https://kafka.apache.org/26/protocol.html#api_versions
+ if conf.ApiVersionsRequest {
+ apiVersionsResponse, err := b.sendAndReceiveApiVersions(3)
+ if err != nil {
+ Logger.Printf("Error while sending ApiVersionsRequest V3 to broker %s: %s\n", b.addr, err)
+ // send a lower version request in case remote cluster is <= 2.4.0.0
+ maxVersion := int16(0)
+ if apiVersionsResponse != nil {
+ for _, k := range apiVersionsResponse.ApiKeys {
+ if k.ApiKey == apiKeyApiVersions {
+ maxVersion = k.MaxVersion
+ break
+ }
+ }
+ }
+ apiVersionsResponse, err = b.sendAndReceiveApiVersions(maxVersion)
+ if err != nil {
+ Logger.Printf("Error while sending ApiVersionsRequest V%d to broker %s: %s\n", maxVersion, b.addr, err)
+ }
+ }
+ if apiVersionsResponse != nil {
+ b.brokerAPIVersions = make(apiVersionMap, len(apiVersionsResponse.ApiKeys))
+ for _, key := range apiVersionsResponse.ApiKeys {
+ b.brokerAPIVersions[key.ApiKey] = &apiVersionRange{
+ minVersion: key.MinVersion,
+ maxVersion: key.MaxVersion,
+ }
+ }
+ }
+ }
+
+ if conf.Net.SASL.Mechanism == SASLTypeOAuth && conf.Net.SASL.Version == SASLHandshakeV0 {
+ conf.Net.SASL.Version = SASLHandshakeV1
+ }
+
+ useSaslV0 := conf.Net.SASL.Version == SASLHandshakeV0
+ if conf.Net.SASL.Enable && useSaslV0 {
+ b.connErr = b.authenticateViaSASLv0()
+
+ if b.connErr != nil {
+ err = b.conn.Close()
+ if err == nil {
+ DebugLogger.Printf("Closed connection to broker %s due to SASL v0 auth error: %s\n", b.addr, b.connErr)
+ } else {
+ Logger.Printf("Error while closing connection to broker %s (due to SASL v0 auth error: %s): %s\n", b.addr, b.connErr, err)
+ }
+ b.conn = nil
+ b.opened.Store(false)
+ return
+ }
+ }
+
+ b.done = make(chan bool)
+ b.responses = make(chan *responsePromise, b.conf.Net.MaxOpenRequests-1)
+
+ go withRecover(b.responseReceiver)
+ if conf.Net.SASL.Enable && !useSaslV0 {
+ b.connErr = b.authenticateViaSASLv1()
+ if b.connErr != nil {
+ close(b.responses)
+ <-b.done
+ err = b.conn.Close()
+ if err == nil {
+ DebugLogger.Printf("Closed connection to broker %s due to SASL v1 auth error: %s\n", b.addr, b.connErr)
+ } else {
+ Logger.Printf("Error while closing connection to broker %s (due to SASL v1 auth error: %s): %s\n", b.addr, b.connErr, err)
+ }
+ b.conn = nil
+ b.opened.Store(false)
+ return
+ }
+ }
+ if b.id >= 0 {
+ DebugLogger.Printf("Connected to broker at %s (registered as #%d)\n", b.addr, b.id)
+ } else {
+ DebugLogger.Printf("Connected to broker at %s (unregistered)\n", b.addr)
+ }
+ })
+
+ return nil
+}
+
+func (b *Broker) ResponseSize() int {
+ b.lock.Lock()
+ defer b.lock.Unlock()
+
+ return len(b.responses)
+}
+
+// Connected returns true if the broker is connected and false otherwise. If the broker is not
+// connected but it had tried to connect, the error from that connection attempt is also returned.
+func (b *Broker) Connected() (bool, error) {
+ b.lock.Lock()
+ defer b.lock.Unlock()
+
+ return b.conn != nil, b.connErr
+}
+
+// TLSConnectionState returns the client's TLS connection state. The second return value is false if this is not a tls connection or the connection has not yet been established.
+func (b *Broker) TLSConnectionState() (state tls.ConnectionState, ok bool) {
+ b.lock.Lock()
+ defer b.lock.Unlock()
+
+ if b.conn == nil {
+ return state, false
+ }
+ conn := b.conn
+ if bconn, ok := b.conn.(*bufConn); ok {
+ conn = bconn.Conn
+ }
+ if tc, ok := conn.(*tls.Conn); ok {
+ return tc.ConnectionState(), true
+ }
+ return state, false
+}
+
+// Close closes the broker resources
+func (b *Broker) Close() error {
+ b.lock.Lock()
+ defer b.lock.Unlock()
+
+ if b.conn == nil {
+ return ErrNotConnected
+ }
+
+ close(b.responses)
+ <-b.done
+
+ err := b.conn.Close()
+
+ b.conn = nil
+ b.connErr = nil
+ b.done = nil
+ b.responses = nil
+
+ b.metricRegistry.UnregisterAll()
+
+ if err == nil {
+ DebugLogger.Printf("Closed connection to broker %s\n", b.addr)
+ } else {
+ Logger.Printf("Error while closing connection to broker %s: %s\n", b.addr, err)
+ }
+ b.opened.Store(false)
+
+ return err
+}
+
+// ID returns the broker ID retrieved from Kafka's metadata, or -1 if that is not known.
+func (b *Broker) ID() int32 {
+ return b.id
+}
+
+// Addr returns the broker address as either retrieved from Kafka's metadata or passed to NewBroker.
+func (b *Broker) Addr() string {
+ return b.addr
+}
+
+// Rack returns the broker's rack as retrieved from Kafka's metadata or the
+// empty string if it is not known. The returned value corresponds to the
+// broker's broker.rack configuration setting. Requires protocol version to be
+// at least v0.10.0.0.
+func (b *Broker) Rack() string {
+ if b.rack == nil {
+ return ""
+ }
+ return *b.rack
+}
+
+// GetMetadata send a metadata request and returns a metadata response or error
+func (b *Broker) GetMetadata(request *MetadataRequest) (*MetadataResponse, error) {
+ response := new(MetadataResponse)
+ response.Version = request.Version // Required to ensure use of the correct response header version
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// GetConsumerMetadata send a consumer metadata request and returns a consumer metadata response or error
+func (b *Broker) GetConsumerMetadata(request *ConsumerMetadataRequest) (*ConsumerMetadataResponse, error) {
+ response := new(ConsumerMetadataResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// FindCoordinator sends a find coordinate request and returns a response or error
+func (b *Broker) FindCoordinator(request *FindCoordinatorRequest) (*FindCoordinatorResponse, error) {
+ response := new(FindCoordinatorResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// GetAvailableOffsets return an offset response or error
+func (b *Broker) GetAvailableOffsets(request *OffsetRequest) (*OffsetResponse, error) {
+ response := new(OffsetResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// ProduceCallback function is called once the produce response has been parsed
+// or could not be read.
+type ProduceCallback func(*ProduceResponse, error)
+
+// AsyncProduce sends a produce request and eventually call the provided callback
+// with a produce response or an error.
+//
+// Waiting for the response is generally not blocking on the contrary to using Produce.
+// If the maximum number of in flight request configured is reached then
+// the request will be blocked till a previous response is received.
+//
+// When configured with RequiredAcks == NoResponse, the callback will not be invoked.
+// If an error is returned because the request could not be sent then the callback
+// will not be invoked either.
+//
+// Make sure not to Close the broker in the callback as it will lead to a deadlock.
+func (b *Broker) AsyncProduce(request *ProduceRequest, cb ProduceCallback) error {
+ b.lock.Lock()
+ defer b.lock.Unlock()
+
+ needAcks := request.RequiredAcks != NoResponse
+ // Use a nil promise when no acks is required
+ var promise *responsePromise
+
+ if needAcks {
+ metricRegistry := b.metricRegistry
+
+ // Create ProduceResponse early to provide the header version
+ res := new(ProduceResponse)
+ promise = &responsePromise{
+ response: res,
+ // Packets will be converted to a ProduceResponse in the responseReceiver goroutine
+ handler: func(packets []byte, err error) {
+ if err != nil {
+ // Failed request
+ cb(nil, err)
+ return
+ }
+
+ if err := versionedDecode(packets, res, request.version(), metricRegistry); err != nil {
+ // Malformed response
+ cb(nil, err)
+ return
+ }
+
+ // Well-formed response
+ b.handleThrottledResponse(res)
+ cb(res, nil)
+ },
+ }
+ }
+
+ return b.sendWithPromise(request, promise)
+}
+
+// Produce returns a produce response or error
+func (b *Broker) Produce(request *ProduceRequest) (*ProduceResponse, error) {
+ var (
+ response *ProduceResponse
+ err error
+ )
+
+ if request.RequiredAcks == NoResponse {
+ err = b.sendAndReceive(request, nil)
+ } else {
+ response = new(ProduceResponse)
+ err = b.sendAndReceive(request, response)
+ }
+
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// Fetch returns a FetchResponse or error
+func (b *Broker) Fetch(request *FetchRequest) (*FetchResponse, error) {
+ defer func() {
+ if b.fetchRate != nil {
+ b.fetchRate.Mark(1)
+ }
+ if b.brokerFetchRate != nil {
+ b.brokerFetchRate.Mark(1)
+ }
+ }()
+
+ response := new(FetchResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// CommitOffset return an Offset commit response or error
+func (b *Broker) CommitOffset(request *OffsetCommitRequest) (*OffsetCommitResponse, error) {
+ response := new(OffsetCommitResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// FetchOffset returns an offset fetch response or error
+func (b *Broker) FetchOffset(request *OffsetFetchRequest) (*OffsetFetchResponse, error) {
+ response := new(OffsetFetchResponse)
+ response.Version = request.Version // needed to handle the two header versions
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// JoinGroup returns a join group response or error
+func (b *Broker) JoinGroup(request *JoinGroupRequest) (*JoinGroupResponse, error) {
+ response := new(JoinGroupResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// SyncGroup returns a sync group response or error
+func (b *Broker) SyncGroup(request *SyncGroupRequest) (*SyncGroupResponse, error) {
+ response := new(SyncGroupResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// LeaveGroup return a leave group response or error
+func (b *Broker) LeaveGroup(request *LeaveGroupRequest) (*LeaveGroupResponse, error) {
+ response := new(LeaveGroupResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// Heartbeat returns a heartbeat response or error
+func (b *Broker) Heartbeat(request *HeartbeatRequest) (*HeartbeatResponse, error) {
+ response := new(HeartbeatResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// ListGroups return a list group response or error
+func (b *Broker) ListGroups(request *ListGroupsRequest) (*ListGroupsResponse, error) {
+ response := new(ListGroupsResponse)
+ response.Version = request.Version // Required to ensure use of the correct response header version
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// DescribeGroups return describe group response or error
+func (b *Broker) DescribeGroups(request *DescribeGroupsRequest) (*DescribeGroupsResponse, error) {
+ response := new(DescribeGroupsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// ApiVersions return api version response or error
+func (b *Broker) ApiVersions(request *ApiVersionsRequest) (*ApiVersionsResponse, error) {
+ response := new(ApiVersionsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// CreateTopics send a create topic request and returns create topic response
+func (b *Broker) CreateTopics(request *CreateTopicsRequest) (*CreateTopicsResponse, error) {
+ response := new(CreateTopicsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// DeleteTopics sends a delete topic request and returns delete topic response
+func (b *Broker) DeleteTopics(request *DeleteTopicsRequest) (*DeleteTopicsResponse, error) {
+ response := new(DeleteTopicsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// CreatePartitions sends a create partition request and returns create
+// partitions response or error
+func (b *Broker) CreatePartitions(request *CreatePartitionsRequest) (*CreatePartitionsResponse, error) {
+ response := new(CreatePartitionsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// AlterPartitionReassignments sends a alter partition reassignments request and
+// returns alter partition reassignments response
+func (b *Broker) AlterPartitionReassignments(request *AlterPartitionReassignmentsRequest) (*AlterPartitionReassignmentsResponse, error) {
+ response := new(AlterPartitionReassignmentsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// ListPartitionReassignments sends a list partition reassignments request and
+// returns list partition reassignments response
+func (b *Broker) ListPartitionReassignments(request *ListPartitionReassignmentsRequest) (*ListPartitionReassignmentsResponse, error) {
+ response := new(ListPartitionReassignmentsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// ElectLeaders sends aa elect leaders request and returns list partitions elect result
+func (b *Broker) ElectLeaders(request *ElectLeadersRequest) (*ElectLeadersResponse, error) {
+ response := new(ElectLeadersResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// DeleteRecords send a request to delete records and return delete record
+// response or error
+func (b *Broker) DeleteRecords(request *DeleteRecordsRequest) (*DeleteRecordsResponse, error) {
+ response := new(DeleteRecordsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// DescribeAcls sends a describe acl request and returns a response or error
+func (b *Broker) DescribeAcls(request *DescribeAclsRequest) (*DescribeAclsResponse, error) {
+ response := new(DescribeAclsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// CreateAcls sends a create acl request and returns a response or error
+func (b *Broker) CreateAcls(request *CreateAclsRequest) (*CreateAclsResponse, error) {
+ response := new(CreateAclsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ errs := make([]error, 0)
+ for _, res := range response.AclCreationResponses {
+ if !errors.Is(res.Err, ErrNoError) {
+ errs = append(errs, res.Err)
+ }
+ }
+
+ if len(errs) > 0 {
+ return response, Wrap(ErrCreateACLs, errs...)
+ }
+
+ return response, nil
+}
+
+// DeleteAcls sends a delete acl request and returns a response or error
+func (b *Broker) DeleteAcls(request *DeleteAclsRequest) (*DeleteAclsResponse, error) {
+ response := new(DeleteAclsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// InitProducerID sends an init producer request and returns a response or error
+func (b *Broker) InitProducerID(request *InitProducerIDRequest) (*InitProducerIDResponse, error) {
+ response := new(InitProducerIDResponse)
+ response.Version = request.version()
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// AddPartitionsToTxn send a request to add partition to txn and returns
+// a response or error
+func (b *Broker) AddPartitionsToTxn(request *AddPartitionsToTxnRequest) (*AddPartitionsToTxnResponse, error) {
+ response := new(AddPartitionsToTxnResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// AddOffsetsToTxn sends a request to add offsets to txn and returns a response
+// or error
+func (b *Broker) AddOffsetsToTxn(request *AddOffsetsToTxnRequest) (*AddOffsetsToTxnResponse, error) {
+ response := new(AddOffsetsToTxnResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// EndTxn sends a request to end txn and returns a response or error
+func (b *Broker) EndTxn(request *EndTxnRequest) (*EndTxnResponse, error) {
+ response := new(EndTxnResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// TxnOffsetCommit sends a request to commit transaction offsets and returns
+// a response or error
+func (b *Broker) TxnOffsetCommit(request *TxnOffsetCommitRequest) (*TxnOffsetCommitResponse, error) {
+ response := new(TxnOffsetCommitResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// DescribeConfigs sends a request to describe config and returns a response or
+// error
+func (b *Broker) DescribeConfigs(request *DescribeConfigsRequest) (*DescribeConfigsResponse, error) {
+ response := new(DescribeConfigsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// AlterConfigs sends a request to alter config and return a response or error
+func (b *Broker) AlterConfigs(request *AlterConfigsRequest) (*AlterConfigsResponse, error) {
+ response := new(AlterConfigsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// IncrementalAlterConfigs sends a request to incremental alter config and return a response or error
+func (b *Broker) IncrementalAlterConfigs(request *IncrementalAlterConfigsRequest) (*IncrementalAlterConfigsResponse, error) {
+ response := new(IncrementalAlterConfigsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// DeleteGroups sends a request to delete groups and returns a response or error
+func (b *Broker) DeleteGroups(request *DeleteGroupsRequest) (*DeleteGroupsResponse, error) {
+ response := new(DeleteGroupsResponse)
+
+ if err := b.sendAndReceive(request, response); err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// DeleteOffsets sends a request to delete group offsets and returns a response or error
+func (b *Broker) DeleteOffsets(request *DeleteOffsetsRequest) (*DeleteOffsetsResponse, error) {
+ response := new(DeleteOffsetsResponse)
+
+ if err := b.sendAndReceive(request, response); err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// DescribeLogDirs sends a request to get the broker's log dir paths and sizes
+func (b *Broker) DescribeLogDirs(request *DescribeLogDirsRequest) (*DescribeLogDirsResponse, error) {
+ response := new(DescribeLogDirsResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// DescribeUserScramCredentials sends a request to get SCRAM users
+func (b *Broker) DescribeUserScramCredentials(req *DescribeUserScramCredentialsRequest) (*DescribeUserScramCredentialsResponse, error) {
+ res := new(DescribeUserScramCredentialsResponse)
+
+ err := b.sendAndReceive(req, res)
+ if err != nil {
+ return nil, err
+ }
+
+ return res, err
+}
+
+func (b *Broker) AlterUserScramCredentials(req *AlterUserScramCredentialsRequest) (*AlterUserScramCredentialsResponse, error) {
+ res := new(AlterUserScramCredentialsResponse)
+
+ err := b.sendAndReceive(req, res)
+ if err != nil {
+ return nil, err
+ }
+
+ return res, nil
+}
+
+// DescribeClientQuotas sends a request to get the broker's quotas
+func (b *Broker) DescribeClientQuotas(request *DescribeClientQuotasRequest) (*DescribeClientQuotasResponse, error) {
+ response := new(DescribeClientQuotasResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// AlterClientQuotas sends a request to alter the broker's quotas
+func (b *Broker) AlterClientQuotas(request *AlterClientQuotasRequest) (*AlterClientQuotasResponse, error) {
+ response := new(AlterClientQuotasResponse)
+
+ err := b.sendAndReceive(request, response)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+// readFull ensures the conn ReadDeadline has been setup before making a
+// call to io.ReadFull
+func (b *Broker) readFull(buf []byte) (n int, err error) {
+ if err := b.conn.SetReadDeadline(time.Now().Add(b.conf.Net.ReadTimeout)); err != nil {
+ return 0, err
+ }
+
+ return io.ReadFull(b.conn, buf)
+}
+
+// write ensures the conn Deadline has been setup before making a
+// call to conn.Write
+func (b *Broker) write(buf []byte) (n int, err error) {
+ now := time.Now()
+ if err := b.conn.SetWriteDeadline(now.Add(b.conf.Net.WriteTimeout)); err != nil {
+ return 0, err
+ }
+ // TLS connections require both read and write deadlines to be set
+ // to avoid handshake indefinite blocking
+ // see https://github.com/golang/go/blob/go1.23.0/src/crypto/tls/conn.go#L1192-L1195
+ if b.conf.Net.TLS.Enable {
+ if err := b.conn.SetReadDeadline(now.Add(b.conf.Net.ReadTimeout)); err != nil {
+ return 0, err
+ }
+ }
+
+ return b.conn.Write(buf)
+}
+
+// b.lock must be held by caller
+//
+// a non-nil res results in a response promise being created
+func (b *Broker) send(req, res protocolBody) (*responsePromise, error) {
+ var promise *responsePromise
+ if res != nil {
+ // Packets or error will be sent to the following channels
+ // once the response is received
+ promise = makeResponsePromise(res)
+ }
+
+ if err := b.sendWithPromise(req, promise); err != nil {
+ return nil, err
+ }
+
+ return promise, nil
+}
+
+func makeResponsePromise(res protocolBody) *responsePromise {
+ promise := &responsePromise{
+ response: res,
+ packets: make(chan []byte),
+ errors: make(chan error),
+ }
+ return promise
+}
+
+// b.lock must be held by caller
+func (b *Broker) sendWithPromise(rb protocolBody, promise *responsePromise) error {
+ if b.conn == nil {
+ if b.connErr != nil {
+ return b.connErr
+ }
+ return ErrNotConnected
+ }
+
+ if b.clientSessionReauthenticationTimeMs > 0 && currentUnixMilli() > b.clientSessionReauthenticationTimeMs {
+ err := b.authenticateViaSASLv1()
+ if err != nil {
+ return err
+ }
+ }
+
+ return b.sendInternal(rb, promise)
+}
+
+// b.lock must be held by caller
+func (b *Broker) sendInternal(rb protocolBody, promise *responsePromise) error {
+ // try restricting API version to ranges advertised by the broker
+ if err := restrictApiVersion(rb, b.brokerAPIVersions); err != nil {
+ return err
+ }
+
+ // response versions must always match their corresponding request's
+ if promise != nil && promise.response != nil {
+ promise.response.setVersion(rb.version())
+ }
+
+ if !b.conf.Version.IsAtLeast(rb.requiredVersion()) {
+ return ErrUnsupportedVersion
+ }
+
+ req := &request{correlationID: b.correlationID, clientID: b.conf.ClientID, body: rb}
+ buf, err := encode(req, b.metricRegistry)
+ if err != nil {
+ return err
+ }
+
+ // check and wait if throttled
+ b.waitIfThrottled()
+
+ requestTime := time.Now()
+ // Will be decremented in responseReceiver (except error or request with NoResponse)
+ b.addRequestInFlightMetrics(1)
+ bytes, err := b.write(buf)
+ b.updateOutgoingCommunicationMetrics(bytes)
+ b.updateProtocolMetrics(rb)
+ if err != nil {
+ b.addRequestInFlightMetrics(-1)
+ return err
+ }
+ b.correlationID++
+
+ if promise == nil {
+ // Record request latency without the response
+ b.updateRequestLatencyAndInFlightMetrics(time.Since(requestTime))
+ return nil
+ }
+
+ promise.requestTime = requestTime
+ promise.correlationID = req.correlationID
+ b.responses <- promise
+
+ return nil
+}
+
+func (b *Broker) sendAndReceive(req protocolBody, res protocolBody) error {
+ b.lock.Lock()
+ defer b.lock.Unlock()
+
+ promise, err := b.send(req, res)
+ if err != nil {
+ return err
+ }
+
+ if promise == nil {
+ return nil
+ }
+
+ err = handleResponsePromise(req, res, promise, b.metricRegistry)
+ if err != nil {
+ return err
+ }
+ if res != nil {
+ b.handleThrottledResponse(res)
+ }
+ return nil
+}
+
+func handleResponsePromise(req protocolBody, res protocolBody, promise *responsePromise, metricRegistry metrics.Registry) error {
+ select {
+ case buf := <-promise.packets:
+ return versionedDecode(buf, res, req.version(), metricRegistry)
+ case err := <-promise.errors:
+ return err
+ }
+}
+
+func (b *Broker) decode(pd packetDecoder, version int16) (err error) {
+ b.id, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ host, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ port, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ if version >= 1 {
+ b.rack, err = pd.getNullableString()
+ if err != nil {
+ return err
+ }
+ }
+
+ b.addr = net.JoinHostPort(host, fmt.Sprint(port))
+ if _, _, err := net.SplitHostPort(b.addr); err != nil {
+ return err
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (b *Broker) encode(pe packetEncoder, version int16) (err error) {
+ host, portstr, err := net.SplitHostPort(b.addr)
+ if err != nil {
+ return err
+ }
+
+ port, err := strconv.ParseInt(portstr, 10, 32)
+ if err != nil {
+ return err
+ }
+
+ pe.putInt32(b.id)
+
+ err = pe.putString(host)
+ if err != nil {
+ return err
+ }
+
+ pe.putInt32(int32(port))
+
+ if version >= 1 {
+ err = pe.putNullableString(b.rack)
+ if err != nil {
+ return err
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (b *Broker) responseReceiver() {
+ var dead error
+
+ for promise := range b.responses {
+ if dead != nil {
+ // This was previously incremented in send() and
+ // we are not calling updateIncomingCommunicationMetrics()
+ b.addRequestInFlightMetrics(-1)
+ promise.handle(nil, dead)
+ continue
+ }
+
+ headerLength := getHeaderLength(promise.response.headerVersion())
+ header := make([]byte, headerLength)
+
+ bytesReadHeader, err := b.readFull(header)
+ requestLatency := time.Since(promise.requestTime)
+ if err != nil {
+ b.updateIncomingCommunicationMetrics(bytesReadHeader, requestLatency)
+ dead = err
+ promise.handle(nil, err)
+ continue
+ }
+
+ decodedHeader := responseHeader{}
+ err = versionedDecode(header, &decodedHeader, promise.response.headerVersion(), b.metricRegistry)
+ if err != nil {
+ b.updateIncomingCommunicationMetrics(bytesReadHeader, requestLatency)
+ dead = err
+ promise.handle(nil, err)
+ continue
+ }
+ if decodedHeader.correlationID != promise.correlationID {
+ b.updateIncomingCommunicationMetrics(bytesReadHeader, requestLatency)
+ // TODO if decoded ID < cur ID, discard until we catch up
+ // TODO if decoded ID > cur ID, save it so when cur ID catches up we have a response
+ dead = PacketDecodingError{fmt.Sprintf("correlation ID didn't match, wanted %d, got %d", promise.correlationID, decodedHeader.correlationID)}
+ promise.handle(nil, dead)
+ continue
+ }
+
+ buf := make([]byte, decodedHeader.length-int32(headerLength)+4)
+ bytesReadBody, err := b.readFull(buf)
+ b.updateIncomingCommunicationMetrics(bytesReadHeader+bytesReadBody, requestLatency)
+ if err != nil {
+ dead = err
+ promise.handle(nil, err)
+ continue
+ }
+
+ promise.handle(buf, nil)
+ }
+ close(b.done)
+}
+
+func getHeaderLength(headerVersion int16) int8 {
+ if headerVersion < 1 {
+ return 8
+ } else {
+ // header contains additional tagged field length (0), we don't support actual tags yet.
+ return 9
+ }
+}
+
+func (b *Broker) sendAndReceiveApiVersions(v int16) (*ApiVersionsResponse, error) {
+ rb := &ApiVersionsRequest{
+ Version: v,
+ ClientSoftwareName: defaultClientSoftwareName,
+ ClientSoftwareVersion: version(),
+ }
+
+ req := &request{correlationID: b.correlationID, clientID: b.conf.ClientID, body: rb}
+ buf, err := encode(req, b.metricRegistry)
+ if err != nil {
+ return nil, err
+ }
+
+ requestTime := time.Now()
+ // Will be decremented in updateIncomingCommunicationMetrics (except error)
+ b.addRequestInFlightMetrics(1)
+ bytes, err := b.write(buf)
+ b.updateOutgoingCommunicationMetrics(bytes)
+ if err != nil {
+ b.addRequestInFlightMetrics(-1)
+ Logger.Printf("Failed to send ApiVersionsRequest V%d to %s: %s\n", v, b.addr, err)
+ return nil, err
+ }
+ b.correlationID++
+
+ // Kafka protocol response structure:
+ // - Message length (4 bytes): Total length of the response excluding this field
+ // - ResponseHeader v0 (4 bytes): Contains correlation ID for request-response matching
+ header := make([]byte, 8)
+ _, err = b.readFull(header)
+ if err != nil {
+ b.addRequestInFlightMetrics(-1)
+ Logger.Printf("Failed to read ApiVersionsResponse V%d header from %s: %s\n", v, b.addr, err)
+ return nil, err
+ }
+
+ length := binary.BigEndian.Uint32(header[:4])
+ // we're not using the correlation ID here, but it is part of the response header
+ // correlationID := binary.BigEndian.Uint32(header[4:])
+
+ payload := make([]byte, length-4)
+ n, err := b.readFull(payload)
+ if err != nil {
+ b.addRequestInFlightMetrics(-1)
+ Logger.Printf("Failed to read ApiVersionsResponse V%d payload from %s: %s\n", v, b.addr, err)
+ return nil, err
+ }
+
+ b.updateIncomingCommunicationMetrics(n+8, time.Since(requestTime))
+ res := &ApiVersionsResponse{Version: rb.version()}
+ err = versionedDecode(payload, res, rb.version(), b.metricRegistry)
+ if err != nil {
+ Logger.Printf("Failed to parse ApiVersionsResponse V%d from %s: %s\n", v, b.addr, err)
+ return nil, err
+ }
+
+ kerr := KError(res.ErrorCode)
+ if kerr != ErrNoError {
+ return res, fmt.Errorf("Error in ApiVersionsResponse V%d from %s: %w", res.Version, b.addr, kerr)
+ }
+
+ DebugLogger.Printf("Completed ApiVersionsRequest V%d to %s. Broker supports %d APIs\n", v, b.addr, len(res.ApiKeys))
+ return res, nil
+}
+
+func (b *Broker) authenticateViaSASLv0() error {
+ switch b.conf.Net.SASL.Mechanism {
+ case SASLTypeSCRAMSHA256, SASLTypeSCRAMSHA512:
+ return b.sendAndReceiveSASLSCRAMv0()
+ case SASLTypeGSSAPI:
+ return b.sendAndReceiveKerberos()
+ default:
+ return b.sendAndReceiveSASLPlainAuthV0()
+ }
+}
+
+func (b *Broker) authenticateViaSASLv1() error {
+ metricRegistry := b.metricRegistry
+ if b.conf.Net.SASL.Handshake {
+ handshakeRequest := &SaslHandshakeRequest{Mechanism: string(b.conf.Net.SASL.Mechanism), Version: b.conf.Net.SASL.Version}
+ handshakeResponse := new(SaslHandshakeResponse)
+ prom := makeResponsePromise(handshakeResponse)
+
+ handshakeErr := b.sendInternal(handshakeRequest, prom)
+ if handshakeErr != nil {
+ Logger.Printf("Error while performing SASL handshake %s: %s\n", b.addr, handshakeErr)
+ return handshakeErr
+ }
+ handshakeErr = handleResponsePromise(handshakeRequest, handshakeResponse, prom, metricRegistry)
+ if handshakeErr != nil {
+ Logger.Printf("Error while handling SASL handshake response %s: %s\n", b.addr, handshakeErr)
+ return handshakeErr
+ }
+
+ if !errors.Is(handshakeResponse.Err, ErrNoError) {
+ return handshakeResponse.Err
+ }
+ }
+
+ authSendReceiver := func(authBytes []byte) (*SaslAuthenticateResponse, error) {
+ authenticateRequest := b.createSaslAuthenticateRequest(authBytes)
+ authenticateResponse := new(SaslAuthenticateResponse)
+ prom := makeResponsePromise(authenticateResponse)
+ authErr := b.sendInternal(authenticateRequest, prom)
+ if authErr != nil {
+ Logger.Printf("Error while performing SASL Auth %s\n", b.addr)
+ return nil, authErr
+ }
+ authErr = handleResponsePromise(authenticateRequest, authenticateResponse, prom, metricRegistry)
+ if authErr != nil {
+ Logger.Printf("Error while performing SASL Auth %s: %s\n", b.addr, authErr)
+ return nil, authErr
+ }
+
+ if !errors.Is(authenticateResponse.Err, ErrNoError) {
+ var err error = authenticateResponse.Err
+ if authenticateResponse.ErrorMessage != nil {
+ err = Wrap(authenticateResponse.Err, errors.New(*authenticateResponse.ErrorMessage))
+ }
+ return nil, err
+ }
+
+ b.computeSaslSessionLifetime(authenticateResponse)
+ return authenticateResponse, nil
+ }
+
+ switch b.conf.Net.SASL.Mechanism {
+ case SASLTypeGSSAPI:
+ b.kerberosAuthenticator.Config = &b.conf.Net.SASL.GSSAPI
+ if b.kerberosAuthenticator.NewKerberosClientFunc == nil {
+ b.kerberosAuthenticator.NewKerberosClientFunc = NewKerberosClient
+ }
+ return b.kerberosAuthenticator.AuthorizeV2(b, authSendReceiver)
+ case SASLTypeOAuth:
+ provider := b.conf.Net.SASL.TokenProvider
+ return b.sendAndReceiveSASLOAuth(authSendReceiver, provider)
+ case SASLTypeSCRAMSHA256, SASLTypeSCRAMSHA512:
+ return b.sendAndReceiveSASLSCRAMv1(authSendReceiver, b.conf.Net.SASL.SCRAMClientGeneratorFunc())
+ default:
+ return b.sendAndReceiveSASLPlainAuthV1(authSendReceiver)
+ }
+}
+
+func (b *Broker) sendAndReceiveKerberos() error {
+ b.kerberosAuthenticator.Config = &b.conf.Net.SASL.GSSAPI
+ if b.kerberosAuthenticator.NewKerberosClientFunc == nil {
+ b.kerberosAuthenticator.NewKerberosClientFunc = NewKerberosClient
+ }
+ return b.kerberosAuthenticator.Authorize(b)
+}
+
+func (b *Broker) sendAndReceiveSASLHandshake(saslType SASLMechanism, version int16) error {
+ rb := &SaslHandshakeRequest{Mechanism: string(saslType), Version: version}
+
+ req := &request{correlationID: b.correlationID, clientID: b.conf.ClientID, body: rb}
+ buf, err := encode(req, b.metricRegistry)
+ if err != nil {
+ return err
+ }
+
+ requestTime := time.Now()
+ // Will be decremented in updateIncomingCommunicationMetrics (except error)
+ b.addRequestInFlightMetrics(1)
+ bytes, err := b.write(buf)
+ b.updateOutgoingCommunicationMetrics(bytes)
+ if err != nil {
+ b.addRequestInFlightMetrics(-1)
+ Logger.Printf("Failed to send SASL handshake %s: %s\n", b.addr, err.Error())
+ return err
+ }
+ b.correlationID++
+
+ header := make([]byte, 8) // response header
+ _, err = b.readFull(header)
+ if err != nil {
+ b.addRequestInFlightMetrics(-1)
+ Logger.Printf("Failed to read SASL handshake header : %s\n", err.Error())
+ return err
+ }
+
+ length := binary.BigEndian.Uint32(header[:4])
+ payload := make([]byte, length-4)
+ n, err := b.readFull(payload)
+ if err != nil {
+ b.addRequestInFlightMetrics(-1)
+ Logger.Printf("Failed to read SASL handshake payload : %s\n", err.Error())
+ return err
+ }
+
+ b.updateIncomingCommunicationMetrics(n+8, time.Since(requestTime))
+ res := &SaslHandshakeResponse{}
+
+ err = versionedDecode(payload, res, 0, b.metricRegistry)
+ if err != nil {
+ Logger.Printf("Failed to parse SASL handshake : %s\n", err.Error())
+ return err
+ }
+
+ if !errors.Is(res.Err, ErrNoError) {
+ Logger.Printf("Invalid SASL Mechanism : %s\n", res.Err.Error())
+ return res.Err
+ }
+
+ DebugLogger.Print("Completed pre-auth SASL handshake. Available mechanisms: ", res.EnabledMechanisms)
+ return nil
+}
+
+//
+// In SASL Plain, Kafka expects the auth header to be in the following format
+// Message format (from https://tools.ietf.org/html/rfc4616):
+//
+// message = [authzid] UTF8NUL authcid UTF8NUL passwd
+// authcid = 1*SAFE ; MUST accept up to 255 octets
+// authzid = 1*SAFE ; MUST accept up to 255 octets
+// passwd = 1*SAFE ; MUST accept up to 255 octets
+// UTF8NUL = %x00 ; UTF-8 encoded NUL character
+//
+// SAFE = UTF1 / UTF2 / UTF3 / UTF4
+// ;; any UTF-8 encoded Unicode character except NUL
+//
+//
+
+// Kafka 0.10.x supported SASL PLAIN/Kerberos via KAFKA-3149 (KIP-43).
+// sendAndReceiveSASLPlainAuthV0 flows the v0 sasl auth NOT wrapped in the kafka protocol
+//
+// With SASL v0 handshake and auth then:
+// When credentials are valid, Kafka returns a 4 byte array of null characters.
+// When credentials are invalid, Kafka closes the connection.
+func (b *Broker) sendAndReceiveSASLPlainAuthV0() error {
+ // default to V0 to allow for backward compatibility when SASL is enabled
+ // but not the handshake
+ if b.conf.Net.SASL.Handshake {
+ handshakeErr := b.sendAndReceiveSASLHandshake(SASLTypePlaintext, b.conf.Net.SASL.Version)
+ if handshakeErr != nil {
+ Logger.Printf("Error while performing SASL handshake %s: %s\n", b.addr, handshakeErr)
+ return handshakeErr
+ }
+ }
+
+ length := len(b.conf.Net.SASL.AuthIdentity) + 1 + len(b.conf.Net.SASL.User) + 1 + len(b.conf.Net.SASL.Password)
+ authBytes := make([]byte, length+4) // 4 byte length header + auth data
+ binary.BigEndian.PutUint32(authBytes, uint32(length))
+ copy(authBytes[4:], b.conf.Net.SASL.AuthIdentity+"\x00"+b.conf.Net.SASL.User+"\x00"+b.conf.Net.SASL.Password)
+
+ requestTime := time.Now()
+ // Will be decremented in updateIncomingCommunicationMetrics (except error)
+ b.addRequestInFlightMetrics(1)
+ bytesWritten, err := b.write(authBytes)
+ b.updateOutgoingCommunicationMetrics(bytesWritten)
+ if err != nil {
+ b.addRequestInFlightMetrics(-1)
+ Logger.Printf("Failed to write SASL auth header to broker %s: %s\n", b.addr, err.Error())
+ return err
+ }
+
+ header := make([]byte, 4)
+ n, err := b.readFull(header)
+ b.updateIncomingCommunicationMetrics(n, time.Since(requestTime))
+ // If the credentials are valid, we would get a 4 byte response filled with null characters.
+ // Otherwise, the broker closes the connection and we get an EOF
+ if err != nil {
+ Logger.Printf("Failed to read response while authenticating with SASL to broker %s: %s\n", b.addr, err.Error())
+ return err
+ }
+
+ DebugLogger.Printf("SASL authentication successful with broker %s:%v - %v\n", b.addr, n, header)
+ return nil
+}
+
+// Kafka 1.x.x onward added a SaslAuthenticate request/response message which
+// wraps the SASL flow in the Kafka protocol, which allows for returning
+// meaningful errors on authentication failure.
+func (b *Broker) sendAndReceiveSASLPlainAuthV1(authSendReceiver func(authBytes []byte) (*SaslAuthenticateResponse, error)) error {
+ authBytes := []byte(b.conf.Net.SASL.AuthIdentity + "\x00" + b.conf.Net.SASL.User + "\x00" + b.conf.Net.SASL.Password)
+ _, err := authSendReceiver(authBytes)
+ return err
+}
+
+func currentUnixMilli() int64 {
+ return time.Now().UnixNano() / int64(time.Millisecond)
+}
+
+// sendAndReceiveSASLOAuth performs the authentication flow as described by KIP-255
+// https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=75968876
+func (b *Broker) sendAndReceiveSASLOAuth(authSendReceiver func(authBytes []byte) (*SaslAuthenticateResponse, error), provider AccessTokenProvider) error {
+ token, err := provider.Token()
+ if err != nil {
+ return err
+ }
+
+ message, err := buildClientFirstMessage(token)
+ if err != nil {
+ return err
+ }
+
+ res, err := authSendReceiver(message)
+ if err != nil {
+ return err
+ }
+ isChallenge := len(res.SaslAuthBytes) > 0
+
+ if isChallenge {
+ // Abort the token exchange. The broker returns the failure code.
+ _, err = authSendReceiver([]byte(`\x01`))
+ }
+ return err
+}
+
+func (b *Broker) sendAndReceiveSASLSCRAMv0() error {
+ if err := b.sendAndReceiveSASLHandshake(b.conf.Net.SASL.Mechanism, SASLHandshakeV0); err != nil {
+ return err
+ }
+
+ scramClient := b.conf.Net.SASL.SCRAMClientGeneratorFunc()
+ if err := scramClient.Begin(b.conf.Net.SASL.User, b.conf.Net.SASL.Password, b.conf.Net.SASL.SCRAMAuthzID); err != nil {
+ return fmt.Errorf("failed to start SCRAM exchange with the server: %w", err)
+ }
+
+ msg, err := scramClient.Step("")
+ if err != nil {
+ return fmt.Errorf("failed to advance the SCRAM exchange: %w", err)
+ }
+
+ for !scramClient.Done() {
+ requestTime := time.Now()
+ // Will be decremented in updateIncomingCommunicationMetrics (except error)
+ b.addRequestInFlightMetrics(1)
+ length := len(msg)
+ authBytes := make([]byte, length+4) // 4 byte length header + auth data
+ binary.BigEndian.PutUint32(authBytes, uint32(length))
+ copy(authBytes[4:], msg)
+ _, err := b.write(authBytes)
+ b.updateOutgoingCommunicationMetrics(length + 4)
+ if err != nil {
+ b.addRequestInFlightMetrics(-1)
+ Logger.Printf("Failed to write SASL auth header to broker %s: %s\n", b.addr, err.Error())
+ return err
+ }
+ b.correlationID++
+ header := make([]byte, 4)
+ _, err = b.readFull(header)
+ if err != nil {
+ b.addRequestInFlightMetrics(-1)
+ Logger.Printf("Failed to read response header while authenticating with SASL to broker %s: %s\n", b.addr, err.Error())
+ return err
+ }
+ payload := make([]byte, int32(binary.BigEndian.Uint32(header)))
+ n, err := b.readFull(payload)
+ if err != nil {
+ b.addRequestInFlightMetrics(-1)
+ Logger.Printf("Failed to read response payload while authenticating with SASL to broker %s: %s\n", b.addr, err.Error())
+ return err
+ }
+ b.updateIncomingCommunicationMetrics(n+4, time.Since(requestTime))
+ msg, err = scramClient.Step(string(payload))
+ if err != nil {
+ Logger.Println("SASL authentication failed", err)
+ return err
+ }
+ }
+
+ DebugLogger.Println("SASL authentication succeeded")
+ return nil
+}
+
+func (b *Broker) sendAndReceiveSASLSCRAMv1(authSendReceiver func(authBytes []byte) (*SaslAuthenticateResponse, error), scramClient SCRAMClient) error {
+ if err := scramClient.Begin(b.conf.Net.SASL.User, b.conf.Net.SASL.Password, b.conf.Net.SASL.SCRAMAuthzID); err != nil {
+ return fmt.Errorf("failed to start SCRAM exchange with the server: %w", err)
+ }
+
+ msg, err := scramClient.Step("")
+ if err != nil {
+ return fmt.Errorf("failed to advance the SCRAM exchange: %w", err)
+ }
+
+ for !scramClient.Done() {
+ res, err := authSendReceiver([]byte(msg))
+ if err != nil {
+ return err
+ }
+
+ msg, err = scramClient.Step(string(res.SaslAuthBytes))
+ if err != nil {
+ Logger.Println("SASL authentication failed", err)
+ return err
+ }
+ }
+
+ DebugLogger.Println("SASL authentication succeeded")
+
+ return nil
+}
+
+func (b *Broker) createSaslAuthenticateRequest(msg []byte) *SaslAuthenticateRequest {
+ authenticateRequest := SaslAuthenticateRequest{SaslAuthBytes: msg}
+ if b.conf.Version.IsAtLeast(V2_2_0_0) {
+ authenticateRequest.Version = 1
+ }
+
+ return &authenticateRequest
+}
+
+// Build SASL/OAUTHBEARER initial client response as described by RFC-7628
+// https://tools.ietf.org/html/rfc7628
+func buildClientFirstMessage(token *AccessToken) ([]byte, error) {
+ var ext string
+
+ if token == nil {
+ return []byte{}, fmt.Errorf("failed to build client first message: token is nil")
+ }
+
+ if len(token.Extensions) > 0 {
+ if _, ok := token.Extensions[SASLExtKeyAuth]; ok {
+ return []byte{}, fmt.Errorf("the extension `%s` is invalid", SASLExtKeyAuth)
+ }
+ ext = "\x01" + mapToString(token.Extensions, "=", "\x01")
+ }
+
+ resp := fmt.Appendf(nil, "n,,\x01auth=Bearer %s%s\x01\x01", token.Token, ext)
+
+ return resp, nil
+}
+
+// mapToString returns a list of key-value pairs ordered by key.
+// keyValSep separates the key from the value. elemSep separates each pair.
+func mapToString(extensions map[string]string, keyValSep string, elemSep string) string {
+ buf := make([]string, 0, len(extensions))
+
+ for k, v := range extensions {
+ buf = append(buf, k+keyValSep+v)
+ }
+
+ sort.Strings(buf)
+
+ return strings.Join(buf, elemSep)
+}
+
+func (b *Broker) computeSaslSessionLifetime(res *SaslAuthenticateResponse) {
+ if res.SessionLifetimeMs > 0 {
+ // Follows the Java Kafka implementation from SaslClientAuthenticator.ReauthInfo#setAuthenticationEndAndSessionReauthenticationTimes
+ // pick a random percentage between 85% and 95% for session re-authentication
+ positiveSessionLifetimeMs := res.SessionLifetimeMs
+ authenticationEndMs := currentUnixMilli()
+ pctWindowFactorToTakeNetworkLatencyAndClockDriftIntoAccount := 0.85
+ pctWindowJitterToAvoidReauthenticationStormAcrossManyChannelsSimultaneously := 0.10
+ pctToUse := pctWindowFactorToTakeNetworkLatencyAndClockDriftIntoAccount + rand.Float64()*pctWindowJitterToAvoidReauthenticationStormAcrossManyChannelsSimultaneously
+ sessionLifetimeMsToUse := int64(float64(positiveSessionLifetimeMs) * pctToUse)
+ DebugLogger.Printf("Session expiration in %d ms and session re-authentication on or after %d ms", positiveSessionLifetimeMs, sessionLifetimeMsToUse)
+ b.clientSessionReauthenticationTimeMs = authenticationEndMs + sessionLifetimeMsToUse
+ } else {
+ b.clientSessionReauthenticationTimeMs = 0
+ }
+}
+
+func (b *Broker) updateIncomingCommunicationMetrics(bytes int, requestLatency time.Duration) {
+ b.updateRequestLatencyAndInFlightMetrics(requestLatency)
+ b.responseRate.Mark(1)
+
+ if b.brokerResponseRate != nil {
+ b.brokerResponseRate.Mark(1)
+ }
+
+ responseSize := int64(bytes)
+ b.incomingByteRate.Mark(responseSize)
+ if b.brokerIncomingByteRate != nil {
+ b.brokerIncomingByteRate.Mark(responseSize)
+ }
+
+ b.responseSize.Update(responseSize)
+ if b.brokerResponseSize != nil {
+ b.brokerResponseSize.Update(responseSize)
+ }
+}
+
+func (b *Broker) updateRequestLatencyAndInFlightMetrics(requestLatency time.Duration) {
+ requestLatencyInMs := int64(requestLatency / time.Millisecond)
+ b.requestLatency.Update(requestLatencyInMs)
+
+ if b.brokerRequestLatency != nil {
+ b.brokerRequestLatency.Update(requestLatencyInMs)
+ }
+
+ b.addRequestInFlightMetrics(-1)
+}
+
+func (b *Broker) addRequestInFlightMetrics(i int64) {
+ b.requestsInFlight.Inc(i)
+ if b.brokerRequestsInFlight != nil {
+ b.brokerRequestsInFlight.Inc(i)
+ }
+}
+
+func (b *Broker) updateOutgoingCommunicationMetrics(bytes int) {
+ b.requestRate.Mark(1)
+ if b.brokerRequestRate != nil {
+ b.brokerRequestRate.Mark(1)
+ }
+
+ requestSize := int64(bytes)
+ b.outgoingByteRate.Mark(requestSize)
+ if b.brokerOutgoingByteRate != nil {
+ b.brokerOutgoingByteRate.Mark(requestSize)
+ }
+
+ b.requestSize.Update(requestSize)
+ if b.brokerRequestSize != nil {
+ b.brokerRequestSize.Update(requestSize)
+ }
+}
+
+func (b *Broker) updateProtocolMetrics(rb protocolBody) {
+ protocolRequestsRate := b.protocolRequestsRate[rb.key()]
+ if protocolRequestsRate == nil {
+ protocolRequestsRate = metrics.GetOrRegisterMeter(fmt.Sprintf("protocol-requests-rate-%d", rb.key()), b.metricRegistry)
+ b.protocolRequestsRate[rb.key()] = protocolRequestsRate
+ }
+ protocolRequestsRate.Mark(1)
+
+ if b.brokerProtocolRequestsRate != nil {
+ brokerProtocolRequestsRate := b.brokerProtocolRequestsRate[rb.key()]
+ if brokerProtocolRequestsRate == nil {
+ brokerProtocolRequestsRate = b.registerMeter(fmt.Sprintf("protocol-requests-rate-%d", rb.key()))
+ b.brokerProtocolRequestsRate[rb.key()] = brokerProtocolRequestsRate
+ }
+ brokerProtocolRequestsRate.Mark(1)
+ }
+}
+
+type throttleSupport interface {
+ throttleTime() time.Duration
+}
+
+func (b *Broker) handleThrottledResponse(resp protocolBody) {
+ throttledResponse, ok := resp.(throttleSupport)
+ if !ok {
+ return
+ }
+ throttleTime := throttledResponse.throttleTime()
+ if throttleTime == time.Duration(0) {
+ return
+ }
+ DebugLogger.Printf(
+ "broker/%d %T throttled %v\n", b.ID(), resp, throttleTime)
+ b.setThrottle(throttleTime)
+ b.updateThrottleMetric(throttleTime)
+}
+
+func (b *Broker) setThrottle(throttleTime time.Duration) {
+ b.throttleTimerLock.Lock()
+ defer b.throttleTimerLock.Unlock()
+ if b.throttleTimer != nil {
+ // if there is an existing timer stop/clear it
+ if !b.throttleTimer.Stop() {
+ <-b.throttleTimer.C
+ }
+ }
+ b.throttleTimer = time.NewTimer(throttleTime)
+}
+
+func (b *Broker) waitIfThrottled() {
+ b.throttleTimerLock.Lock()
+ defer b.throttleTimerLock.Unlock()
+ if b.throttleTimer != nil {
+ DebugLogger.Printf("broker/%d waiting for throttle timer\n", b.ID())
+ <-b.throttleTimer.C
+ b.throttleTimer = nil
+ }
+}
+
+func (b *Broker) updateThrottleMetric(throttleTime time.Duration) {
+ if b.brokerThrottleTime != nil {
+ throttleTimeInMs := int64(throttleTime / time.Millisecond)
+ b.brokerThrottleTime.Update(throttleTimeInMs)
+ }
+}
+
+func (b *Broker) registerMetrics() {
+ b.brokerIncomingByteRate = b.registerMeter("incoming-byte-rate")
+ b.brokerRequestRate = b.registerMeter("request-rate")
+ b.brokerFetchRate = b.registerMeter("consumer-fetch-rate")
+ b.brokerRequestSize = b.registerHistogram("request-size")
+ b.brokerRequestLatency = b.registerHistogram("request-latency-in-ms")
+ b.brokerOutgoingByteRate = b.registerMeter("outgoing-byte-rate")
+ b.brokerResponseRate = b.registerMeter("response-rate")
+ b.brokerResponseSize = b.registerHistogram("response-size")
+ b.brokerRequestsInFlight = b.registerCounter("requests-in-flight")
+ b.brokerThrottleTime = b.registerHistogram("throttle-time-in-ms")
+ b.brokerProtocolRequestsRate = map[int16]metrics.Meter{}
+}
+
+func (b *Broker) registerMeter(name string) metrics.Meter {
+ nameForBroker := getMetricNameForBroker(name, b)
+ return metrics.GetOrRegisterMeter(nameForBroker, b.metricRegistry)
+}
+
+func (b *Broker) registerHistogram(name string) metrics.Histogram {
+ nameForBroker := getMetricNameForBroker(name, b)
+ return getOrRegisterHistogram(nameForBroker, b.metricRegistry)
+}
+
+func (b *Broker) registerCounter(name string) metrics.Counter {
+ nameForBroker := getMetricNameForBroker(name, b)
+ return metrics.GetOrRegisterCounter(nameForBroker, b.metricRegistry)
+}
+
+func validServerNameTLS(addr string, cfg *tls.Config) *tls.Config {
+ if cfg == nil {
+ cfg = &tls.Config{
+ MinVersion: tls.VersionTLS12,
+ }
+ }
+ if cfg.ServerName != "" {
+ return cfg
+ }
+
+ c := cfg.Clone()
+ sn, _, err := net.SplitHostPort(addr)
+ if err != nil {
+ Logger.Println(fmt.Errorf("failed to get ServerName from addr %w", err))
+ }
+ c.ServerName = sn
+ return c
+}
diff --git a/vendor/github.com/IBM/sarama/client.go b/vendor/github.com/IBM/sarama/client.go
new file mode 100644
index 0000000..0dc29e2
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/client.go
@@ -0,0 +1,1291 @@
+package sarama
+
+import (
+ "context"
+ "errors"
+ "math"
+ "math/rand"
+ "net"
+ "slices"
+ "sort"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "golang.org/x/net/proxy"
+)
+
+// Client is a generic Kafka client. It manages connections to one or more Kafka brokers.
+// You MUST call Close() on a client to avoid leaks, it will not be garbage-collected
+// automatically when it passes out of scope. It is safe to share a client amongst many
+// users, however Kafka will process requests from a single client strictly in serial,
+// so it is generally more efficient to use the default one client per producer/consumer.
+type Client interface {
+ // Config returns the Config struct of the client. This struct should not be
+ // altered after it has been created.
+ Config() *Config
+
+ // Controller returns the cluster controller broker. It will return a
+ // locally cached value if it's available. You can call RefreshController
+ // to update the cached value. Requires Kafka 0.10 or higher.
+ Controller() (*Broker, error)
+
+ // RefreshController retrieves the cluster controller from fresh metadata
+ // and stores it in the local cache. Requires Kafka 0.10 or higher.
+ RefreshController() (*Broker, error)
+
+ // Brokers returns the current set of active brokers as retrieved from cluster metadata.
+ Brokers() []*Broker
+
+ // Broker returns the active Broker if available for the broker ID.
+ Broker(brokerID int32) (*Broker, error)
+
+ // Topics returns the set of available topics as retrieved from cluster metadata.
+ Topics() ([]string, error)
+
+ // Partitions returns the sorted list of all partition IDs for the given topic.
+ Partitions(topic string) ([]int32, error)
+
+ // WritablePartitions returns the sorted list of all writable partition IDs for
+ // the given topic, where "writable" means "having a valid leader accepting
+ // writes".
+ WritablePartitions(topic string) ([]int32, error)
+
+ // Leader returns the broker object that is the leader of the current
+ // topic/partition, as determined by querying the cluster metadata.
+ Leader(topic string, partitionID int32) (*Broker, error)
+
+ // LeaderAndEpoch returns the leader and its epoch for the current
+ // topic/partition, as determined by querying the cluster metadata.
+ LeaderAndEpoch(topic string, partitionID int32) (*Broker, int32, error)
+
+ // Replicas returns the set of all replica IDs for the given partition.
+ Replicas(topic string, partitionID int32) ([]int32, error)
+
+ // InSyncReplicas returns the set of all in-sync replica IDs for the given
+ // partition. In-sync replicas are replicas which are fully caught up with
+ // the partition leader.
+ InSyncReplicas(topic string, partitionID int32) ([]int32, error)
+
+ // OfflineReplicas returns the set of all offline replica IDs for the given
+ // partition. Offline replicas are replicas which are offline
+ OfflineReplicas(topic string, partitionID int32) ([]int32, error)
+
+ // RefreshBrokers takes a list of addresses to be used as seed brokers.
+ // Existing broker connections are closed and the updated list of seed brokers
+ // will be used for the next metadata fetch.
+ RefreshBrokers(addrs []string) error
+
+ // RefreshMetadata takes a list of topics and queries the cluster to refresh the
+ // available metadata for those topics. If no topics are provided, it will refresh
+ // metadata for all topics.
+ RefreshMetadata(topics ...string) error
+
+ // GetOffset queries the cluster to get the most recent available offset at the
+ // given time (in milliseconds) on the topic/partition combination.
+ // Time should be OffsetOldest for the earliest available offset,
+ // OffsetNewest for the offset of the message that will be produced next, or a time.
+ GetOffset(topic string, partitionID int32, time int64) (int64, error)
+
+ // Coordinator returns the coordinating broker for a consumer group. It will
+ // return a locally cached value if it's available. You can call
+ // RefreshCoordinator to update the cached value. This function only works on
+ // Kafka 0.8.2 and higher.
+ Coordinator(consumerGroup string) (*Broker, error)
+
+ // RefreshCoordinator retrieves the coordinator for a consumer group and stores it
+ // in local cache. This function only works on Kafka 0.8.2 and higher.
+ RefreshCoordinator(consumerGroup string) error
+
+ // TransactionCoordinator returns the coordinating broker for a transaction id. It will
+ // return a locally cached value if it's available. You can call
+ // RefreshCoordinator to update the cached value. This function only works on
+ // Kafka 0.11.0.0 and higher.
+ TransactionCoordinator(transactionID string) (*Broker, error)
+
+ // RefreshTransactionCoordinator retrieves the coordinator for a transaction id and stores it
+ // in local cache. This function only works on Kafka 0.11.0.0 and higher.
+ RefreshTransactionCoordinator(transactionID string) error
+
+ // InitProducerID retrieves information required for Idempotent Producer
+ InitProducerID() (*InitProducerIDResponse, error)
+
+ // LeastLoadedBroker retrieves broker that has the least responses pending
+ LeastLoadedBroker() *Broker
+
+ // PartitionNotReadable checks if partition is not readable
+ PartitionNotReadable(topic string, partition int32) bool
+
+ // Close shuts down all broker connections managed by this client. It is required
+ // to call this function before a client object passes out of scope, as it will
+ // otherwise leak memory. You must close any Producers or Consumers using a client
+ // before you close the client.
+ Close() error
+
+ // Closed returns true if the client has already had Close called on it
+ Closed() bool
+}
+
+const (
+ // OffsetNewest stands for the log head offset, i.e. the offset that will be
+ // assigned to the next message that will be produced to the partition. You
+ // can send this to a client's GetOffset method to get this offset, or when
+ // calling ConsumePartition to start consuming new messages.
+ OffsetNewest int64 = -1
+ // OffsetOldest stands for the oldest offset available on the broker for a
+ // partition. You can send this to a client's GetOffset method to get this
+ // offset, or when calling ConsumePartition to start consuming from the
+ // oldest offset that is still available on the broker.
+ OffsetOldest int64 = -2
+)
+
+type client struct {
+ // updateMetadataMs stores the time at which metadata was lasted updated.
+ // Note: this accessed atomically so must be the first word in the struct
+ // as per golang/go#41970
+ updateMetadataMs atomic.Int64
+
+ conf *Config
+ closer, closed chan none // for shutting down background metadata updater
+
+ // the broker addresses given to us through the constructor are not guaranteed to be returned in
+ // the cluster metadata (I *think* it only returns brokers who are currently leading partitions?)
+ // so we store them separately
+ seedBrokers []*Broker
+ deadSeeds []*Broker
+
+ controllerID int32 // cluster controller broker id
+ brokers map[int32]*Broker // maps broker ids to brokers
+ metadata map[string]map[int32]*PartitionMetadata // maps topics to partition ids to metadata
+ metadataTopics map[string]none // topics that need to collect metadata
+ coordinators map[string]int32 // Maps consumer group names to coordinating broker IDs
+ transactionCoordinators map[string]int32 // Maps transaction ids to coordinating broker IDs
+
+ // If the number of partitions is large, we can get some churn calling cachedPartitions,
+ // so the result is cached. It is important to update this value whenever metadata is changed
+ cachedPartitionsResults map[string][maxPartitionIndex][]int32
+
+ lock sync.RWMutex // protects access to the maps that hold cluster state.
+
+ metadataRefresh metadataRefresh
+}
+
+// NewClient creates a new Client. It connects to one of the given broker addresses
+// and uses that broker to automatically fetch metadata on the rest of the kafka cluster. If metadata cannot
+// be retrieved from any of the given broker addresses, the client is not created.
+func NewClient(addrs []string, conf *Config) (Client, error) {
+ DebugLogger.Println("Initializing new client")
+
+ if conf == nil {
+ conf = NewConfig()
+ }
+
+ if err := conf.Validate(); err != nil {
+ return nil, err
+ }
+
+ if len(addrs) < 1 {
+ return nil, ConfigurationError("You must provide at least one broker address")
+ }
+
+ if strings.Contains(addrs[0], ".servicebus.windows.net") {
+ if conf.Version.IsAtLeast(V1_1_0_0) || !conf.Version.IsAtLeast(V0_11_0_0) {
+ Logger.Println("Connecting to Azure Event Hubs, forcing version to V1_0_0_0 for compatibility")
+ conf.Version = V1_0_0_0
+ }
+ }
+ client := &client{
+ conf: conf,
+ closer: make(chan none),
+ closed: make(chan none),
+ brokers: make(map[int32]*Broker),
+ metadata: make(map[string]map[int32]*PartitionMetadata),
+ metadataTopics: make(map[string]none),
+ cachedPartitionsResults: make(map[string][maxPartitionIndex][]int32),
+ coordinators: make(map[string]int32),
+ transactionCoordinators: make(map[string]int32),
+ }
+ refresh := func(topics []string) error {
+ deadline := time.Time{}
+ if client.conf.Metadata.Timeout > 0 {
+ deadline = time.Now().Add(client.conf.Metadata.Timeout)
+ }
+ return client.tryRefreshMetadata(topics, client.conf.Metadata.Retry.Max, deadline)
+ }
+ if conf.Metadata.SingleFlight {
+ client.metadataRefresh = newSingleFlightRefresher(refresh)
+ } else {
+ client.metadataRefresh = refresh
+ }
+
+ if conf.Net.ResolveCanonicalBootstrapServers {
+ var err error
+ addrs, err = client.resolveCanonicalNames(addrs)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ client.randomizeSeedBrokers(addrs)
+
+ if conf.Metadata.Full {
+ // do an initial fetch of all cluster metadata by specifying an empty list of topics
+ err := client.RefreshMetadata()
+ if err == nil {
+ } else if errors.Is(err, ErrLeaderNotAvailable) || errors.Is(err, ErrReplicaNotAvailable) || errors.Is(err, ErrTopicAuthorizationFailed) || errors.Is(err, ErrClusterAuthorizationFailed) {
+ // indicates that maybe part of the cluster is down, but is not fatal to creating the client
+ Logger.Println(err)
+ } else {
+ close(client.closed) // we haven't started the background updater yet, so we have to do this manually
+ _ = client.Close()
+ return nil, err
+ }
+ }
+ go withRecover(client.backgroundMetadataUpdater)
+
+ DebugLogger.Println("Successfully initialized new client")
+
+ return client, nil
+}
+
+func (client *client) Config() *Config {
+ return client.conf
+}
+
+func (client *client) Brokers() []*Broker {
+ client.lock.RLock()
+ defer client.lock.RUnlock()
+ brokers := make([]*Broker, 0, len(client.brokers))
+ for _, broker := range client.brokers {
+ brokers = append(brokers, broker)
+ }
+ return brokers
+}
+
+func (client *client) Broker(brokerID int32) (*Broker, error) {
+ client.lock.RLock()
+ defer client.lock.RUnlock()
+ broker, ok := client.brokers[brokerID]
+ if !ok {
+ return nil, ErrBrokerNotFound
+ }
+ _ = broker.Open(client.conf)
+ return broker, nil
+}
+
+func (client *client) InitProducerID() (*InitProducerIDResponse, error) {
+ // FIXME: this InitProducerID seems to only be called from client_test.go (TestInitProducerIDConnectionRefused) and has been superceded by transaction_manager.go?
+ brokerErrors := make([]error, 0)
+ for broker := client.LeastLoadedBroker(); broker != nil; broker = client.LeastLoadedBroker() {
+ request := &InitProducerIDRequest{}
+
+ if client.conf.Version.IsAtLeast(V2_7_0_0) {
+ // Version 4 adds the support for new error code PRODUCER_FENCED.
+ request.Version = 4
+ } else if client.conf.Version.IsAtLeast(V2_5_0_0) {
+ // Version 3 adds ProducerId and ProducerEpoch, allowing producers to try to resume after an INVALID_PRODUCER_EPOCH error
+ request.Version = 3
+ } else if client.conf.Version.IsAtLeast(V2_4_0_0) {
+ // Version 2 is the first flexible version.
+ request.Version = 2
+ } else if client.conf.Version.IsAtLeast(V2_0_0_0) {
+ // Version 1 is the same as version 0.
+ request.Version = 1
+ }
+
+ response, err := broker.InitProducerID(request)
+ if err == nil {
+ return response, nil
+ } else {
+ // some error, remove that broker and try again
+ Logger.Printf("Client got error from broker %d when issuing InitProducerID : %v\n", broker.ID(), err)
+ _ = broker.Close()
+ brokerErrors = append(brokerErrors, err)
+ client.deregisterBroker(broker)
+ }
+ }
+
+ return nil, Wrap(ErrOutOfBrokers, brokerErrors...)
+}
+
+func (client *client) Close() error {
+ if client.Closed() {
+ // Chances are this is being called from a defer() and the error will go unobserved
+ // so we go ahead and log the event in this case.
+ Logger.Printf("Close() called on already closed client")
+ return ErrClosedClient
+ }
+
+ // shutdown and wait for the background thread before we take the lock, to avoid races
+ close(client.closer)
+ <-client.closed
+
+ client.lock.Lock()
+ defer client.lock.Unlock()
+ DebugLogger.Println("Closing Client")
+
+ for _, broker := range client.brokers {
+ safeAsyncClose(broker)
+ }
+
+ for _, broker := range client.seedBrokers {
+ safeAsyncClose(broker)
+ }
+
+ client.brokers = nil
+ client.metadata = nil
+ client.metadataTopics = nil
+
+ return nil
+}
+
+func (client *client) Closed() bool {
+ client.lock.RLock()
+ defer client.lock.RUnlock()
+
+ return client.brokers == nil
+}
+
+func (client *client) Topics() ([]string, error) {
+ if client.Closed() {
+ return nil, ErrClosedClient
+ }
+
+ client.lock.RLock()
+ defer client.lock.RUnlock()
+
+ ret := make([]string, 0, len(client.metadata))
+ for topic := range client.metadata {
+ ret = append(ret, topic)
+ }
+
+ return ret, nil
+}
+
+func (client *client) MetadataTopics() ([]string, error) {
+ if client.Closed() {
+ return nil, ErrClosedClient
+ }
+
+ client.lock.RLock()
+ defer client.lock.RUnlock()
+
+ ret := make([]string, 0, len(client.metadataTopics))
+ for topic := range client.metadataTopics {
+ ret = append(ret, topic)
+ }
+
+ return ret, nil
+}
+
+func (client *client) Partitions(topic string) ([]int32, error) {
+ return client.getPartitions(topic, allPartitions)
+}
+
+func (client *client) WritablePartitions(topic string) ([]int32, error) {
+ return client.getPartitions(topic, writablePartitions)
+}
+
+func (client *client) getPartitions(topic string, pt partitionType) ([]int32, error) {
+ if client.Closed() {
+ return nil, ErrClosedClient
+ }
+
+ partitions := client.cachedPartitions(topic, pt)
+
+ // len==0 catches when it's nil (no such topic) and the odd case when every single
+ // partition is undergoing leader election simultaneously. Callers have to be able to handle
+ // this function returning an empty slice (which is a valid return value) but catching it
+ // here the first time (note we *don't* catch it below where we return ErrUnknownTopicOrPartition) triggers
+ // a metadata refresh as a nicety so callers can just try again and don't have to manually
+ // trigger a refresh (otherwise they'd just keep getting a stale cached copy).
+ if len(partitions) == 0 {
+ err := client.RefreshMetadata(topic)
+ if err != nil {
+ return nil, err
+ }
+ partitions = client.cachedPartitions(topic, pt)
+ }
+
+ if partitions == nil {
+ return nil, ErrUnknownTopicOrPartition
+ }
+
+ return partitions, nil
+}
+
+func (client *client) Replicas(topic string, partitionID int32) ([]int32, error) {
+ return client.getReplicas(topic, partitionID, func(metadata *PartitionMetadata) []int32 {
+ return metadata.Replicas
+ })
+}
+
+func (client *client) InSyncReplicas(topic string, partitionID int32) ([]int32, error) {
+ return client.getReplicas(topic, partitionID, func(metadata *PartitionMetadata) []int32 {
+ return metadata.Isr
+ })
+}
+
+func (client *client) OfflineReplicas(topic string, partitionID int32) ([]int32, error) {
+ return client.getReplicas(topic, partitionID, func(metadata *PartitionMetadata) []int32 {
+ return metadata.OfflineReplicas
+ })
+}
+
+func (client *client) getReplicas(topic string, partitionID int32, extractor func(metadata *PartitionMetadata) []int32) ([]int32, error) {
+ if client.Closed() {
+ return nil, ErrClosedClient
+ }
+
+ metadata := client.cachedMetadata(topic, partitionID)
+
+ if metadata == nil {
+ err := client.RefreshMetadata(topic)
+ if err != nil {
+ return nil, err
+ }
+ metadata = client.cachedMetadata(topic, partitionID)
+ }
+
+ if metadata == nil {
+ return nil, ErrUnknownTopicOrPartition
+ }
+
+ replicas := extractor(metadata)
+ if errors.Is(metadata.Err, ErrReplicaNotAvailable) {
+ return dupInt32Slice(replicas), metadata.Err
+ }
+ return dupInt32Slice(replicas), nil
+}
+
+func (client *client) Leader(topic string, partitionID int32) (*Broker, error) {
+ leader, _, err := client.LeaderAndEpoch(topic, partitionID)
+ return leader, err
+}
+
+func (client *client) LeaderAndEpoch(topic string, partitionID int32) (*Broker, int32, error) {
+ if client.Closed() {
+ return nil, -1, ErrClosedClient
+ }
+
+ leader, epoch, err := client.cachedLeader(topic, partitionID)
+ if leader == nil {
+ err = client.RefreshMetadata(topic)
+ if err != nil {
+ return nil, -1, err
+ }
+ leader, epoch, err = client.cachedLeader(topic, partitionID)
+ }
+
+ return leader, epoch, err
+}
+
+func (client *client) RefreshBrokers(addrs []string) error {
+ if client.Closed() {
+ return ErrClosedClient
+ }
+
+ client.lock.Lock()
+ defer client.lock.Unlock()
+
+ for _, broker := range client.brokers {
+ safeAsyncClose(broker)
+ }
+ client.brokers = make(map[int32]*Broker)
+
+ for _, broker := range client.seedBrokers {
+ safeAsyncClose(broker)
+ }
+
+ for _, broker := range client.deadSeeds {
+ safeAsyncClose(broker)
+ }
+
+ client.seedBrokers = nil
+ client.deadSeeds = nil
+
+ client.randomizeSeedBrokers(addrs)
+
+ return nil
+}
+
+func (client *client) RefreshMetadata(topics ...string) error {
+ if client.Closed() {
+ return ErrClosedClient
+ }
+
+ // Prior to 0.8.2, Kafka will throw exceptions on an empty topic and not return a proper
+ // error. This handles the case by returning an error instead of sending it
+ // off to Kafka. See: https://github.com/IBM/sarama/pull/38#issuecomment-26362310
+ if slices.Contains(topics, "") {
+ return ErrInvalidTopic // this is the error that 0.8.2 and later correctly return
+ }
+ return client.metadataRefresh(topics)
+}
+
+func (client *client) GetOffset(topic string, partitionID int32, timestamp int64) (int64, error) {
+ if client.Closed() {
+ return -1, ErrClosedClient
+ }
+
+ offset, err := client.getOffset(topic, partitionID, timestamp)
+ if err != nil {
+ if err := client.RefreshMetadata(topic); err != nil {
+ return -1, err
+ }
+ return client.getOffset(topic, partitionID, timestamp)
+ }
+
+ return offset, err
+}
+
+func (client *client) Controller() (*Broker, error) {
+ if client.Closed() {
+ return nil, ErrClosedClient
+ }
+
+ if !client.conf.Version.IsAtLeast(V0_10_0_0) {
+ return nil, ErrUnsupportedVersion
+ }
+
+ controller := client.cachedController()
+ if controller == nil {
+ if err := client.refreshMetadata(); err != nil {
+ return nil, err
+ }
+ controller = client.cachedController()
+ }
+
+ if controller == nil {
+ return nil, ErrControllerNotAvailable
+ }
+
+ _ = controller.Open(client.conf)
+ return controller, nil
+}
+
+// deregisterController removes the cached controllerID
+func (client *client) deregisterController() {
+ client.lock.Lock()
+ defer client.lock.Unlock()
+ if controller, ok := client.brokers[client.controllerID]; ok {
+ _ = controller.Close()
+ delete(client.brokers, client.controllerID)
+ }
+}
+
+// RefreshController retrieves the cluster controller from fresh metadata
+// and stores it in the local cache. Requires Kafka 0.10 or higher.
+func (client *client) RefreshController() (*Broker, error) {
+ if client.Closed() {
+ return nil, ErrClosedClient
+ }
+
+ client.deregisterController()
+
+ if err := client.refreshMetadata(); err != nil {
+ return nil, err
+ }
+
+ controller := client.cachedController()
+ if controller == nil {
+ return nil, ErrControllerNotAvailable
+ }
+
+ _ = controller.Open(client.conf)
+ return controller, nil
+}
+
+func (client *client) Coordinator(consumerGroup string) (*Broker, error) {
+ if client.Closed() {
+ return nil, ErrClosedClient
+ }
+
+ coordinator := client.cachedCoordinator(consumerGroup)
+
+ if coordinator == nil {
+ if err := client.RefreshCoordinator(consumerGroup); err != nil {
+ return nil, err
+ }
+ coordinator = client.cachedCoordinator(consumerGroup)
+ }
+
+ if coordinator == nil {
+ return nil, ErrConsumerCoordinatorNotAvailable
+ }
+
+ _ = coordinator.Open(client.conf)
+ return coordinator, nil
+}
+
+func (client *client) RefreshCoordinator(consumerGroup string) error {
+ if client.Closed() {
+ return ErrClosedClient
+ }
+
+ response, err := client.findCoordinator(consumerGroup, CoordinatorGroup, client.conf.Metadata.Retry.Max)
+ if err != nil {
+ return err
+ }
+
+ client.lock.Lock()
+ defer client.lock.Unlock()
+ client.registerBroker(response.Coordinator)
+ client.coordinators[consumerGroup] = response.Coordinator.ID()
+ return nil
+}
+
+func (client *client) TransactionCoordinator(transactionID string) (*Broker, error) {
+ if client.Closed() {
+ return nil, ErrClosedClient
+ }
+
+ coordinator := client.cachedTransactionCoordinator(transactionID)
+
+ if coordinator == nil {
+ if err := client.RefreshTransactionCoordinator(transactionID); err != nil {
+ return nil, err
+ }
+ coordinator = client.cachedTransactionCoordinator(transactionID)
+ }
+
+ if coordinator == nil {
+ return nil, ErrConsumerCoordinatorNotAvailable
+ }
+
+ _ = coordinator.Open(client.conf)
+ return coordinator, nil
+}
+
+func (client *client) RefreshTransactionCoordinator(transactionID string) error {
+ if client.Closed() {
+ return ErrClosedClient
+ }
+
+ response, err := client.findCoordinator(transactionID, CoordinatorTransaction, client.conf.Metadata.Retry.Max)
+ if err != nil {
+ return err
+ }
+
+ client.lock.Lock()
+ defer client.lock.Unlock()
+ client.registerBroker(response.Coordinator)
+ client.transactionCoordinators[transactionID] = response.Coordinator.ID()
+ return nil
+}
+
+// private broker management helpers
+
+func (client *client) randomizeSeedBrokers(addrs []string) {
+ random := rand.New(rand.NewSource(time.Now().UnixNano()))
+ for _, index := range random.Perm(len(addrs)) {
+ client.seedBrokers = append(client.seedBrokers, NewBroker(addrs[index]))
+ }
+}
+
+func (client *client) updateBroker(brokers []*Broker) {
+ currentBroker := make(map[int32]*Broker, len(brokers))
+
+ for _, broker := range brokers {
+ currentBroker[broker.ID()] = broker
+ if client.brokers[broker.ID()] == nil { // add new broker
+ client.brokers[broker.ID()] = broker
+ DebugLogger.Printf("client/brokers registered new broker #%d at %s", broker.ID(), broker.Addr())
+ } else if broker.Addr() != client.brokers[broker.ID()].Addr() { // replace broker with new address
+ safeAsyncClose(client.brokers[broker.ID()])
+ client.brokers[broker.ID()] = broker
+ Logger.Printf("client/brokers replaced registered broker #%d with %s", broker.ID(), broker.Addr())
+ }
+ }
+
+ for id, broker := range client.brokers {
+ if _, exist := currentBroker[id]; !exist { // remove old broker
+ safeAsyncClose(broker)
+ delete(client.brokers, id)
+ Logger.Printf("client/broker remove invalid broker #%d with %s", broker.ID(), broker.Addr())
+ }
+ }
+}
+
+// registerBroker makes sure a broker received by a Metadata or Coordinator request is registered
+// in the brokers map. It returns the broker that is registered, which may be the provided broker,
+// or a previously registered Broker instance. You must hold the write lock before calling this function.
+func (client *client) registerBroker(broker *Broker) {
+ if client.brokers == nil {
+ Logger.Printf("cannot register broker #%d at %s, client already closed", broker.ID(), broker.Addr())
+ return
+ }
+
+ if client.brokers[broker.ID()] == nil {
+ client.brokers[broker.ID()] = broker
+ DebugLogger.Printf("client/brokers registered new broker #%d at %s", broker.ID(), broker.Addr())
+ } else if broker.Addr() != client.brokers[broker.ID()].Addr() {
+ safeAsyncClose(client.brokers[broker.ID()])
+ client.brokers[broker.ID()] = broker
+ Logger.Printf("client/brokers replaced registered broker #%d with %s", broker.ID(), broker.Addr())
+ }
+}
+
+// deregisterBroker removes a broker from the broker list, and if it's
+// not in the broker list, removes it from seedBrokers.
+func (client *client) deregisterBroker(broker *Broker) {
+ client.lock.Lock()
+ defer client.lock.Unlock()
+
+ _, ok := client.brokers[broker.ID()]
+ if ok {
+ Logger.Printf("client/brokers deregistered broker #%d at %s", broker.ID(), broker.Addr())
+ delete(client.brokers, broker.ID())
+ return
+ }
+ if len(client.seedBrokers) > 0 && broker == client.seedBrokers[0] {
+ client.deadSeeds = append(client.deadSeeds, broker)
+ client.seedBrokers = client.seedBrokers[1:]
+ }
+}
+
+func (client *client) resurrectDeadBrokers() {
+ client.lock.Lock()
+ defer client.lock.Unlock()
+
+ Logger.Printf("client/brokers resurrecting %d dead seed brokers", len(client.deadSeeds))
+ client.seedBrokers = append(client.seedBrokers, client.deadSeeds...)
+ client.deadSeeds = nil
+}
+
+// LeastLoadedBroker returns the broker with the least pending requests.
+// Firstly, choose the broker from cached broker list. If the broker list is empty, choose from seed brokers.
+func (client *client) LeastLoadedBroker() *Broker {
+ client.lock.RLock()
+ defer client.lock.RUnlock()
+
+ var leastLoadedBroker *Broker
+ pendingRequests := math.MaxInt
+ for _, broker := range client.brokers {
+ if pendingRequests > broker.ResponseSize() {
+ pendingRequests = broker.ResponseSize()
+ leastLoadedBroker = broker
+ }
+ }
+ if leastLoadedBroker != nil {
+ _ = leastLoadedBroker.Open(client.conf)
+ return leastLoadedBroker
+ }
+
+ if len(client.seedBrokers) > 0 {
+ _ = client.seedBrokers[0].Open(client.conf)
+ return client.seedBrokers[0]
+ }
+
+ return leastLoadedBroker
+}
+
+// private caching/lazy metadata helpers
+
+type partitionType int
+
+const (
+ allPartitions partitionType = iota
+ writablePartitions
+ // If you add any more types, update the partition cache in update()
+
+ // Ensure this is the last partition type value
+ maxPartitionIndex
+)
+
+func (client *client) cachedMetadata(topic string, partitionID int32) *PartitionMetadata {
+ client.lock.RLock()
+ defer client.lock.RUnlock()
+
+ partitions := client.metadata[topic]
+ if partitions != nil {
+ return partitions[partitionID]
+ }
+
+ return nil
+}
+
+func (client *client) cachedPartitions(topic string, partitionSet partitionType) []int32 {
+ client.lock.RLock()
+ defer client.lock.RUnlock()
+
+ partitions, exists := client.cachedPartitionsResults[topic]
+
+ if !exists {
+ return nil
+ }
+ return partitions[partitionSet]
+}
+
+func (client *client) setPartitionCache(topic string, partitionSet partitionType) []int32 {
+ partitions := client.metadata[topic]
+
+ if partitions == nil {
+ return nil
+ }
+
+ ret := make([]int32, 0, len(partitions))
+ for _, partition := range partitions {
+ if partitionSet == writablePartitions && errors.Is(partition.Err, ErrLeaderNotAvailable) {
+ continue
+ }
+ ret = append(ret, partition.ID)
+ }
+
+ sort.Sort(int32Slice(ret))
+ return ret
+}
+
+func (client *client) cachedLeader(topic string, partitionID int32) (*Broker, int32, error) {
+ client.lock.RLock()
+ defer client.lock.RUnlock()
+
+ partitions := client.metadata[topic]
+ if partitions != nil {
+ metadata, ok := partitions[partitionID]
+ if ok {
+ if errors.Is(metadata.Err, ErrLeaderNotAvailable) {
+ return nil, -1, ErrLeaderNotAvailable
+ }
+ b := client.brokers[metadata.Leader]
+ if b == nil {
+ return nil, -1, ErrLeaderNotAvailable
+ }
+ _ = b.Open(client.conf)
+ return b, metadata.LeaderEpoch, nil
+ }
+ }
+
+ return nil, -1, ErrUnknownTopicOrPartition
+}
+
+func (client *client) getOffset(topic string, partitionID int32, timestamp int64) (int64, error) {
+ broker, err := client.Leader(topic, partitionID)
+ if err != nil {
+ return -1, err
+ }
+
+ request := NewOffsetRequest(client.conf.Version)
+ request.AddBlock(topic, partitionID, timestamp, 1)
+
+ response, err := broker.GetAvailableOffsets(request)
+ if err != nil {
+ _ = broker.Close()
+ return -1, err
+ }
+
+ block := response.GetBlock(topic, partitionID)
+ if block == nil {
+ _ = broker.Close()
+ return -1, ErrIncompleteResponse
+ }
+ if !errors.Is(block.Err, ErrNoError) {
+ return -1, block.Err
+ }
+ if len(block.Offsets) != 1 {
+ return -1, ErrOffsetOutOfRange
+ }
+
+ return block.Offsets[0], nil
+}
+
+// core metadata update logic
+
+func (client *client) backgroundMetadataUpdater() {
+ defer close(client.closed)
+
+ if client.conf.Metadata.RefreshFrequency == time.Duration(0) {
+ return
+ }
+
+ ticker := time.NewTicker(client.conf.Metadata.RefreshFrequency)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ticker.C:
+ if err := client.refreshMetadata(); err != nil {
+ Logger.Println("Client background metadata update:", err)
+ }
+ case <-client.closer:
+ return
+ }
+ }
+}
+
+func (client *client) refreshMetadata() error {
+ var topics []string
+
+ if !client.conf.Metadata.Full {
+ if specificTopics, err := client.MetadataTopics(); err != nil {
+ return err
+ } else if len(specificTopics) == 0 {
+ return ErrNoTopicsToUpdateMetadata
+ } else {
+ topics = specificTopics
+ }
+ }
+
+ if err := client.RefreshMetadata(topics...); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (client *client) tryRefreshMetadata(topics []string, attemptsRemaining int, deadline time.Time) error {
+ pastDeadline := func(backoff time.Duration) bool {
+ if !deadline.IsZero() && time.Now().Add(backoff).After(deadline) {
+ // we are past the deadline
+ return true
+ }
+ return false
+ }
+ retry := func(err error) error {
+ if attemptsRemaining > 0 {
+ backoff := client.computeBackoff(attemptsRemaining)
+ if pastDeadline(backoff) {
+ Logger.Println("client/metadata skipping last retries as we would go past the metadata timeout")
+ return err
+ }
+ if backoff > 0 {
+ time.Sleep(backoff)
+ }
+
+ t := client.updateMetadataMs.Load()
+ if time.Since(time.UnixMilli(t)) < backoff {
+ return err
+ }
+ attemptsRemaining--
+ Logger.Printf("client/metadata retrying after %dms... (%d attempts remaining)\n", backoff/time.Millisecond, attemptsRemaining)
+
+ return client.tryRefreshMetadata(topics, attemptsRemaining, deadline)
+ }
+ return err
+ }
+
+ broker := client.LeastLoadedBroker()
+ brokerErrors := make([]error, 0)
+ for ; broker != nil && !pastDeadline(0); broker = client.LeastLoadedBroker() {
+ allowAutoTopicCreation := client.conf.Metadata.AllowAutoTopicCreation
+ if len(topics) > 0 {
+ DebugLogger.Printf("client/metadata fetching metadata for %v from broker %s\n", topics, broker.addr)
+ } else {
+ allowAutoTopicCreation = false
+ DebugLogger.Printf("client/metadata fetching metadata for all topics from broker %s\n", broker.addr)
+ }
+
+ req := NewMetadataRequest(client.conf.Version, topics)
+ req.AllowAutoTopicCreation = allowAutoTopicCreation
+ client.updateMetadataMs.Store(time.Now().UnixMilli())
+
+ response, err := broker.GetMetadata(req)
+ var kerror KError
+ var packetEncodingError PacketEncodingError
+ if err == nil {
+ // When talking to the startup phase of a broker, it is possible to receive an empty metadata set. We should remove that broker and try next broker (https://issues.apache.org/jira/browse/KAFKA-7924).
+ if len(response.Brokers) == 0 {
+ Logger.Printf("client/metadata receiving empty brokers from the metadata response when requesting the broker #%d at %s", broker.ID(), broker.addr)
+ _ = broker.Close()
+ client.deregisterBroker(broker)
+ continue
+ }
+ allKnownMetaData := len(topics) == 0
+ // valid response, use it
+ shouldRetry, err := client.updateMetadata(response, allKnownMetaData)
+ if shouldRetry {
+ Logger.Println("client/metadata found some partitions to be leaderless")
+ return retry(err) // note: err can be nil
+ }
+ return err
+ } else if errors.As(err, &packetEncodingError) {
+ // didn't even send, return the error
+ return err
+ } else if errors.As(err, &kerror) {
+ // if SASL auth error return as this _should_ be a non retryable err for all brokers
+ if errors.Is(err, ErrSASLAuthenticationFailed) {
+ Logger.Println("client/metadata failed SASL authentication")
+ return err
+ }
+
+ if errors.Is(err, ErrTopicAuthorizationFailed) {
+ Logger.Println("client is not authorized to access this topic. The topics were: ", topics)
+ return err
+ }
+ // else remove that broker and try again
+ Logger.Printf("client/metadata got error from broker %d while fetching metadata: %v\n", broker.ID(), err)
+ _ = broker.Close()
+ client.deregisterBroker(broker)
+ } else {
+ // some other error, remove that broker and try again
+ Logger.Printf("client/metadata got error from broker %d while fetching metadata: %v\n", broker.ID(), err)
+ brokerErrors = append(brokerErrors, err)
+ _ = broker.Close()
+ client.deregisterBroker(broker)
+ }
+ }
+
+ error := Wrap(ErrOutOfBrokers, brokerErrors...)
+ if broker != nil {
+ Logger.Printf("client/metadata not fetching metadata from broker %s as we would go past the metadata timeout\n", broker.addr)
+ return retry(error)
+ }
+
+ Logger.Println("client/metadata no available broker to send metadata request to")
+ client.resurrectDeadBrokers()
+ return retry(error)
+}
+
+// if no fatal error, returns a list of topics that need retrying due to ErrLeaderNotAvailable
+func (client *client) updateMetadata(data *MetadataResponse, allKnownMetaData bool) (retry bool, err error) {
+ if client.Closed() {
+ return
+ }
+
+ client.lock.Lock()
+ defer client.lock.Unlock()
+
+ // For all the brokers we received:
+ // - if it is a new ID, save it
+ // - if it is an existing ID, but the address we have is stale, discard the old one and save it
+ // - if some brokers is not exist in it, remove old broker
+ // - otherwise ignore it, replacing our existing one would just bounce the connection
+ client.updateBroker(data.Brokers)
+
+ client.controllerID = data.ControllerID
+
+ if allKnownMetaData {
+ client.metadata = make(map[string]map[int32]*PartitionMetadata)
+ client.metadataTopics = make(map[string]none)
+ client.cachedPartitionsResults = make(map[string][maxPartitionIndex][]int32)
+ }
+ for _, topic := range data.Topics {
+ // topics must be added firstly to `metadataTopics` to guarantee that all
+ // requested topics must be recorded to keep them trackable for periodically
+ // metadata refresh.
+ if _, exists := client.metadataTopics[topic.Name]; !exists {
+ client.metadataTopics[topic.Name] = none{}
+ }
+ delete(client.metadata, topic.Name)
+ delete(client.cachedPartitionsResults, topic.Name)
+
+ switch topic.Err {
+ case ErrNoError:
+ // no-op
+ case ErrInvalidTopic, ErrTopicAuthorizationFailed: // don't retry, don't store partial results
+ err = topic.Err
+ continue
+ case ErrUnknownTopicOrPartition: // retry, do not store partial partition results
+ err = topic.Err
+ retry = true
+ continue
+ case ErrLeaderNotAvailable: // retry, but store partial partition results
+ retry = true
+ default: // don't retry, don't store partial results
+ Logger.Printf("Unexpected topic-level metadata error: %s", topic.Err)
+ err = topic.Err
+ continue
+ }
+
+ client.metadata[topic.Name] = make(map[int32]*PartitionMetadata, len(topic.Partitions))
+ for _, partition := range topic.Partitions {
+ client.metadata[topic.Name][partition.ID] = partition
+ if errors.Is(partition.Err, ErrLeaderNotAvailable) {
+ retry = true
+ }
+ }
+
+ var partitionCache [maxPartitionIndex][]int32
+ partitionCache[allPartitions] = client.setPartitionCache(topic.Name, allPartitions)
+ partitionCache[writablePartitions] = client.setPartitionCache(topic.Name, writablePartitions)
+ client.cachedPartitionsResults[topic.Name] = partitionCache
+ }
+
+ return
+}
+
+func (client *client) cachedCoordinator(consumerGroup string) *Broker {
+ client.lock.RLock()
+ defer client.lock.RUnlock()
+ if coordinatorID, ok := client.coordinators[consumerGroup]; ok {
+ return client.brokers[coordinatorID]
+ }
+ return nil
+}
+
+func (client *client) cachedTransactionCoordinator(transactionID string) *Broker {
+ client.lock.RLock()
+ defer client.lock.RUnlock()
+ if coordinatorID, ok := client.transactionCoordinators[transactionID]; ok {
+ return client.brokers[coordinatorID]
+ }
+ return nil
+}
+
+func (client *client) cachedController() *Broker {
+ client.lock.RLock()
+ defer client.lock.RUnlock()
+
+ return client.brokers[client.controllerID]
+}
+
+func (client *client) computeBackoff(attemptsRemaining int) time.Duration {
+ if client.conf.Metadata.Retry.BackoffFunc != nil {
+ maxRetries := client.conf.Metadata.Retry.Max
+ retries := maxRetries - attemptsRemaining
+ return client.conf.Metadata.Retry.BackoffFunc(retries, maxRetries)
+ }
+ return client.conf.Metadata.Retry.Backoff
+}
+
+func (client *client) findCoordinator(coordinatorKey string, coordinatorType CoordinatorType, attemptsRemaining int) (*FindCoordinatorResponse, error) {
+ retry := func(err error) (*FindCoordinatorResponse, error) {
+ if attemptsRemaining > 0 {
+ backoff := client.computeBackoff(attemptsRemaining)
+ attemptsRemaining--
+ Logger.Printf("client/coordinator retrying after %dms... (%d attempts remaining)\n", backoff/time.Millisecond, attemptsRemaining)
+ time.Sleep(backoff)
+ return client.findCoordinator(coordinatorKey, coordinatorType, attemptsRemaining)
+ }
+ return nil, err
+ }
+
+ brokerErrors := make([]error, 0)
+ for broker := client.LeastLoadedBroker(); broker != nil; broker = client.LeastLoadedBroker() {
+ DebugLogger.Printf("client/coordinator requesting coordinator for %s from %s\n", coordinatorKey, broker.Addr())
+
+ request := new(FindCoordinatorRequest)
+ request.CoordinatorKey = coordinatorKey
+ request.CoordinatorType = coordinatorType
+
+ // Version 1 adds KeyType.
+ if client.conf.Version.IsAtLeast(V0_11_0_0) {
+ request.Version = 1
+ }
+ // Version 2 is the same as version 1.
+ if client.conf.Version.IsAtLeast(V2_0_0_0) {
+ request.Version = 2
+ }
+
+ response, err := broker.FindCoordinator(request)
+ if err != nil {
+ Logger.Printf("client/coordinator request to broker %s failed: %s\n", broker.Addr(), err)
+
+ var packetEncodingError PacketEncodingError
+ if errors.As(err, &packetEncodingError) {
+ return nil, err
+ } else {
+ _ = broker.Close()
+ brokerErrors = append(brokerErrors, err)
+ client.deregisterBroker(broker)
+ continue
+ }
+ }
+
+ if errors.Is(response.Err, ErrNoError) {
+ DebugLogger.Printf("client/coordinator coordinator for %s is #%d (%s)\n", coordinatorKey, response.Coordinator.ID(), response.Coordinator.Addr())
+ return response, nil
+ } else if errors.Is(response.Err, ErrConsumerCoordinatorNotAvailable) {
+ Logger.Printf("client/coordinator coordinator for %s is not available\n", coordinatorKey)
+
+ // This is very ugly, but this scenario will only happen once per cluster.
+ // The __consumer_offsets topic only has to be created one time.
+ // The number of partitions not configurable, but partition 0 should always exist.
+ if _, err := client.Leader("__consumer_offsets", 0); err != nil {
+ Logger.Printf("client/coordinator the __consumer_offsets topic is not initialized completely yet. Waiting 2 seconds...\n")
+ time.Sleep(2 * time.Second)
+ }
+ if coordinatorType == CoordinatorTransaction {
+ if _, err := client.Leader("__transaction_state", 0); err != nil {
+ Logger.Printf("client/coordinator the __transaction_state topic is not initialized completely yet. Waiting 2 seconds...\n")
+ time.Sleep(2 * time.Second)
+ }
+ }
+
+ return retry(ErrConsumerCoordinatorNotAvailable)
+ } else if errors.Is(response.Err, ErrGroupAuthorizationFailed) {
+ Logger.Printf("client was not authorized to access group %s while attempting to find coordinator", coordinatorKey)
+ return retry(ErrGroupAuthorizationFailed)
+ } else {
+ return nil, response.Err
+ }
+ }
+
+ Logger.Println("client/coordinator no available broker to send consumer metadata request to")
+ client.resurrectDeadBrokers()
+ return retry(Wrap(ErrOutOfBrokers, brokerErrors...))
+}
+
+func (client *client) resolveCanonicalNames(addrs []string) ([]string, error) {
+ ctx := context.Background()
+
+ dialer := client.Config().getDialer()
+ resolver := net.Resolver{
+ Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+ // dial func should only be called once, so switching within is acceptable
+ switch d := dialer.(type) {
+ case proxy.ContextDialer:
+ return d.DialContext(ctx, network, address)
+ default:
+ // we have no choice but to ignore the context
+ return d.Dial(network, address)
+ }
+ },
+ }
+
+ canonicalAddrs := make(map[string]struct{}, len(addrs)) // dedupe as we go
+ for _, addr := range addrs {
+ host, port, err := net.SplitHostPort(addr)
+ if err != nil {
+ return nil, err // message includes addr
+ }
+
+ ips, err := resolver.LookupHost(ctx, host)
+ if err != nil {
+ return nil, err // message includes host
+ }
+ for _, ip := range ips {
+ ptrs, err := resolver.LookupAddr(ctx, ip)
+ if err != nil {
+ return nil, err // message includes ip
+ }
+
+ // unlike the Java client, we do not further check that PTRs resolve
+ ptr := strings.TrimSuffix(ptrs[0], ".") // trailing dot breaks GSSAPI
+ canonicalAddrs[net.JoinHostPort(ptr, port)] = struct{}{}
+ }
+ }
+
+ addrs = make([]string, 0, len(canonicalAddrs))
+ for addr := range canonicalAddrs {
+ addrs = append(addrs, addr)
+ }
+ return addrs, nil
+}
+
+// nopCloserClient embeds an existing Client, but disables
+// the Close method (yet all other methods pass
+// through unchanged). This is for use in larger structs
+// where it is undesirable to close the client that was
+// passed in by the caller.
+type nopCloserClient struct {
+ Client
+}
+
+// Close intercepts and purposely does not call the underlying
+// client's Close() method.
+func (ncc *nopCloserClient) Close() error {
+ return nil
+}
+
+func (client *client) PartitionNotReadable(topic string, partition int32) bool {
+ client.lock.RLock()
+ defer client.lock.RUnlock()
+
+ pm := client.metadata[topic][partition]
+ if pm == nil {
+ return true
+ }
+ return pm.Leader == -1
+}
diff --git a/vendor/github.com/IBM/sarama/compress.go b/vendor/github.com/IBM/sarama/compress.go
new file mode 100644
index 0000000..b752cb8
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/compress.go
@@ -0,0 +1,191 @@
+package sarama
+
+import (
+ "bytes"
+ "fmt"
+ "sync"
+
+ snappy "github.com/eapache/go-xerial-snappy"
+ "github.com/klauspost/compress/gzip"
+ "github.com/pierrec/lz4/v4"
+)
+
+var (
+ lz4WriterPool = sync.Pool{
+ New: func() interface{} {
+ lz := lz4.NewWriter(nil)
+ if err := lz.Apply(lz4.BlockSizeOption(lz4.Block64Kb)); err != nil {
+ panic(err)
+ }
+ return lz
+ },
+ }
+
+ gzipWriterPool = sync.Pool{
+ New: func() interface{} {
+ return gzip.NewWriter(nil)
+ },
+ }
+ gzipWriterPoolForCompressionLevel1 = sync.Pool{
+ New: func() interface{} {
+ gz, err := gzip.NewWriterLevel(nil, 1)
+ if err != nil {
+ panic(err)
+ }
+ return gz
+ },
+ }
+ gzipWriterPoolForCompressionLevel2 = sync.Pool{
+ New: func() interface{} {
+ gz, err := gzip.NewWriterLevel(nil, 2)
+ if err != nil {
+ panic(err)
+ }
+ return gz
+ },
+ }
+ gzipWriterPoolForCompressionLevel3 = sync.Pool{
+ New: func() interface{} {
+ gz, err := gzip.NewWriterLevel(nil, 3)
+ if err != nil {
+ panic(err)
+ }
+ return gz
+ },
+ }
+ gzipWriterPoolForCompressionLevel4 = sync.Pool{
+ New: func() interface{} {
+ gz, err := gzip.NewWriterLevel(nil, 4)
+ if err != nil {
+ panic(err)
+ }
+ return gz
+ },
+ }
+ gzipWriterPoolForCompressionLevel5 = sync.Pool{
+ New: func() interface{} {
+ gz, err := gzip.NewWriterLevel(nil, 5)
+ if err != nil {
+ panic(err)
+ }
+ return gz
+ },
+ }
+ gzipWriterPoolForCompressionLevel6 = sync.Pool{
+ New: func() interface{} {
+ gz, err := gzip.NewWriterLevel(nil, 6)
+ if err != nil {
+ panic(err)
+ }
+ return gz
+ },
+ }
+ gzipWriterPoolForCompressionLevel7 = sync.Pool{
+ New: func() interface{} {
+ gz, err := gzip.NewWriterLevel(nil, 7)
+ if err != nil {
+ panic(err)
+ }
+ return gz
+ },
+ }
+ gzipWriterPoolForCompressionLevel8 = sync.Pool{
+ New: func() interface{} {
+ gz, err := gzip.NewWriterLevel(nil, 8)
+ if err != nil {
+ panic(err)
+ }
+ return gz
+ },
+ }
+ gzipWriterPoolForCompressionLevel9 = sync.Pool{
+ New: func() interface{} {
+ gz, err := gzip.NewWriterLevel(nil, 9)
+ if err != nil {
+ panic(err)
+ }
+ return gz
+ },
+ }
+)
+
+func compress(cc CompressionCodec, level int, data []byte) ([]byte, error) {
+ switch cc {
+ case CompressionNone:
+ return data, nil
+ case CompressionGZIP:
+ return gzipCompress(level, data)
+ case CompressionSnappy:
+ return snappy.Encode(data), nil
+ case CompressionLZ4:
+ return lz4Compress(data)
+ case CompressionZSTD:
+ return zstdCompress(ZstdEncoderParams{level}, nil, data)
+ default:
+ return nil, PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", cc)}
+ }
+}
+
+func gzipCompress(level int, data []byte) ([]byte, error) {
+ var (
+ buf bytes.Buffer
+ writer *gzip.Writer
+ pool *sync.Pool
+ )
+
+ switch level {
+ case CompressionLevelDefault:
+ pool = &gzipWriterPool
+ case 1:
+ pool = &gzipWriterPoolForCompressionLevel1
+ case 2:
+ pool = &gzipWriterPoolForCompressionLevel2
+ case 3:
+ pool = &gzipWriterPoolForCompressionLevel3
+ case 4:
+ pool = &gzipWriterPoolForCompressionLevel4
+ case 5:
+ pool = &gzipWriterPoolForCompressionLevel5
+ case 6:
+ pool = &gzipWriterPoolForCompressionLevel6
+ case 7:
+ pool = &gzipWriterPoolForCompressionLevel7
+ case 8:
+ pool = &gzipWriterPoolForCompressionLevel8
+ case 9:
+ pool = &gzipWriterPoolForCompressionLevel9
+ default:
+ var err error
+ writer, err = gzip.NewWriterLevel(&buf, level)
+ if err != nil {
+ return nil, err
+ }
+ }
+ if pool != nil {
+ writer = pool.Get().(*gzip.Writer)
+ writer.Reset(&buf)
+ defer pool.Put(writer)
+ }
+ if _, err := writer.Write(data); err != nil {
+ return nil, err
+ }
+ if err := writer.Close(); err != nil {
+ return nil, err
+ }
+ return buf.Bytes(), nil
+}
+
+func lz4Compress(data []byte) ([]byte, error) {
+ writer := lz4WriterPool.Get().(*lz4.Writer)
+ defer lz4WriterPool.Put(writer)
+
+ var buf bytes.Buffer
+ writer.Reset(&buf)
+ if _, err := writer.Write(data); err != nil {
+ return nil, err
+ }
+ if err := writer.Close(); err != nil {
+ return nil, err
+ }
+ return buf.Bytes(), nil
+}
diff --git a/vendor/github.com/IBM/sarama/config.go b/vendor/github.com/IBM/sarama/config.go
new file mode 100644
index 0000000..5bac2b5
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/config.go
@@ -0,0 +1,917 @@
+package sarama
+
+import (
+ "crypto/tls"
+ "fmt"
+ "io"
+ "net"
+ "regexp"
+ "time"
+
+ "github.com/klauspost/compress/gzip"
+ "github.com/rcrowley/go-metrics"
+ "golang.org/x/net/proxy"
+)
+
+const defaultClientID = "sarama"
+
+// validClientID specifies the permitted characters for a client.id when
+// connecting to Kafka versions before 1.0.0 (KIP-190)
+var validClientID = regexp.MustCompile(`\A[A-Za-z0-9._-]+\z`)
+
+// Config is used to pass multiple configuration options to Sarama's constructors.
+type Config struct {
+ // Admin is the namespace for ClusterAdmin properties used by the administrative Kafka client.
+ Admin struct {
+ Retry struct {
+ // The total number of times to retry sending (retriable) admin requests (default 5).
+ // Similar to the `retries` setting of the JVM AdminClientConfig.
+ Max int
+ // Backoff time between retries of a failed request (default 100ms)
+ Backoff time.Duration
+ }
+ // The maximum duration the administrative Kafka client will wait for ClusterAdmin operations,
+ // including topics, brokers, configurations and ACLs (defaults to 3 seconds).
+ Timeout time.Duration
+ }
+
+ // Net is the namespace for network-level properties used by the Broker, and
+ // shared by the Client/Producer/Consumer.
+ Net struct {
+ // How many outstanding requests a connection is allowed to have before
+ // sending on it blocks (default 5).
+ // Throughput can improve but message ordering is not guaranteed if Producer.Idempotent is disabled, see:
+ // https://kafka.apache.org/protocol#protocol_network
+ // https://kafka.apache.org/28/documentation.html#producerconfigs_max.in.flight.requests.per.connection
+ MaxOpenRequests int
+
+ // All three of the below configurations are similar to the
+ // `socket.timeout.ms` setting in JVM kafka. All of them default
+ // to 30 seconds.
+ DialTimeout time.Duration // How long to wait for the initial connection.
+ ReadTimeout time.Duration // How long to wait for a response.
+ WriteTimeout time.Duration // How long to wait for a transmit.
+
+ // ResolveCanonicalBootstrapServers turns each bootstrap broker address
+ // into a set of IPs, then does a reverse lookup on each one to get its
+ // canonical hostname. This list of hostnames then replaces the
+ // original address list. Similar to the `client.dns.lookup` option in
+ // the JVM client, this is especially useful with GSSAPI, where it
+ // allows providing an alias record instead of individual broker
+ // hostnames. Defaults to false.
+ ResolveCanonicalBootstrapServers bool
+
+ TLS struct {
+ // Whether or not to use TLS when connecting to the broker
+ // (defaults to false).
+ Enable bool
+ // The TLS configuration to use for secure connections if
+ // enabled (defaults to nil).
+ Config *tls.Config
+ }
+
+ // SASL based authentication with broker. While there are multiple SASL authentication methods
+ // the current implementation is limited to plaintext (SASL/PLAIN) authentication
+ SASL struct {
+ // Whether or not to use SASL authentication when connecting to the broker
+ // (defaults to false).
+ Enable bool
+ // SASLMechanism is the name of the enabled SASL mechanism.
+ // Possible values: OAUTHBEARER, PLAIN (defaults to PLAIN).
+ Mechanism SASLMechanism
+ // Version is the SASL Protocol Version to use
+ // Kafka > 1.x should use V1, except on Azure EventHub which use V0
+ Version int16
+ // Whether or not to send the Kafka SASL handshake first if enabled
+ // (defaults to true). You should only set this to false if you're using
+ // a non-Kafka SASL proxy.
+ Handshake bool
+ // AuthIdentity is an (optional) authorization identity (authzid) to
+ // use for SASL/PLAIN authentication (if different from User) when
+ // an authenticated user is permitted to act as the presented
+ // alternative user. See RFC4616 for details.
+ AuthIdentity string
+ // User is the authentication identity (authcid) to present for
+ // SASL/PLAIN or SASL/SCRAM authentication
+ User string
+ // Password for SASL/PLAIN authentication
+ Password string
+ // authz id used for SASL/SCRAM authentication
+ SCRAMAuthzID string
+ // SCRAMClientGeneratorFunc is a generator of a user provided implementation of a SCRAM
+ // client used to perform the SCRAM exchange with the server.
+ SCRAMClientGeneratorFunc func() SCRAMClient
+ // TokenProvider is a user-defined callback for generating
+ // access tokens for SASL/OAUTHBEARER auth. See the
+ // AccessTokenProvider interface docs for proper implementation
+ // guidelines.
+ TokenProvider AccessTokenProvider
+
+ GSSAPI GSSAPIConfig
+ }
+
+ // KeepAlive specifies the keep-alive period for an active network connection (defaults to 0).
+ // If zero or positive, keep-alives are enabled.
+ // If negative, keep-alives are disabled.
+ KeepAlive time.Duration
+
+ // LocalAddr is the local address to use when dialing an
+ // address. The address must be of a compatible type for the
+ // network being dialed.
+ // If nil, a local address is automatically chosen.
+ LocalAddr net.Addr
+
+ Proxy struct {
+ // Whether or not to use proxy when connecting to the broker
+ // (defaults to false).
+ Enable bool
+ // The proxy dialer to use enabled (defaults to nil).
+ Dialer proxy.Dialer
+ }
+ }
+
+ // Metadata is the namespace for metadata management properties used by the
+ // Client, and shared by the Producer/Consumer.
+ Metadata struct {
+ Retry struct {
+ // The total number of times to retry a metadata request when the
+ // cluster is in the middle of a leader election (default 3).
+ Max int
+ // How long to wait for leader election to occur before retrying
+ // (default 250ms). Similar to the JVM's `retry.backoff.ms`.
+ Backoff time.Duration
+ // Called to compute backoff time dynamically. Useful for implementing
+ // more sophisticated backoff strategies. This takes precedence over
+ // `Backoff` if set.
+ BackoffFunc func(retries, maxRetries int) time.Duration
+ }
+ // How frequently to refresh the cluster metadata in the background.
+ // Defaults to 10 minutes. Set to 0 to disable. Similar to
+ // `topic.metadata.refresh.interval.ms` in the JVM version.
+ RefreshFrequency time.Duration
+
+ // Whether to maintain a full set of metadata for all topics, or just
+ // the minimal set that has been necessary so far. The full set is simpler
+ // and usually more convenient, but can take up a substantial amount of
+ // memory if you have many topics and partitions. Defaults to true.
+ Full bool
+
+ // How long to wait for a successful metadata response.
+ // Disabled by default which means a metadata request against an unreachable
+ // cluster (all brokers are unreachable or unresponsive) can take up to
+ // `Net.[Dial|Read]Timeout * BrokerCount * (Metadata.Retry.Max + 1) + Metadata.Retry.Backoff * Metadata.Retry.Max`
+ // to fail.
+ Timeout time.Duration
+
+ // Whether to allow auto-create topics in metadata refresh. If set to true,
+ // the broker may auto-create topics that we requested which do not already exist,
+ // if it is configured to do so (`auto.create.topics.enable` is true). Defaults to true.
+ AllowAutoTopicCreation bool
+
+ // SingleFlight controls whether to send a single metadata refresh request at a given time
+ // or whether to allow anyone to refresh the metadata concurrently.
+ // If this is set to true and the client needs to refresh the metadata from different goroutines,
+ // the requests will be batched together so that a single refresh is sent at a time.
+ // See https://github.com/IBM/sarama/issues/3224 for more details.
+ // SingleFlight defaults to true.
+ SingleFlight bool
+ }
+
+ // Producer is the namespace for configuration related to producing messages,
+ // used by the Producer.
+ Producer struct {
+ // The maximum permitted size of a message (defaults to 1000000). Should be
+ // set equal to or smaller than the broker's `message.max.bytes`.
+ MaxMessageBytes int
+ // The level of acknowledgement reliability needed from the broker (defaults
+ // to WaitForLocal). Equivalent to the `request.required.acks` setting of the
+ // JVM producer.
+ RequiredAcks RequiredAcks
+ // The maximum duration the broker will wait the receipt of the number of
+ // RequiredAcks (defaults to 10 seconds). This is only relevant when
+ // RequiredAcks is set to WaitForAll or a number > 1. Only supports
+ // millisecond resolution, nanoseconds will be truncated. Equivalent to
+ // the JVM producer's `request.timeout.ms` setting.
+ Timeout time.Duration
+ // The type of compression to use on messages (defaults to no compression).
+ // Similar to `compression.codec` setting of the JVM producer.
+ Compression CompressionCodec
+ // The level of compression to use on messages. The meaning depends
+ // on the actual compression type used and defaults to default compression
+ // level for the codec.
+ CompressionLevel int
+ // Generates partitioners for choosing the partition to send messages to
+ // (defaults to hashing the message key). Similar to the `partitioner.class`
+ // setting for the JVM producer.
+ Partitioner PartitionerConstructor
+ // If enabled, the producer will ensure that exactly one copy of each message is
+ // written.
+ Idempotent bool
+ // Transaction specify
+ Transaction struct {
+ // Used in transactions to identify an instance of a producer through restarts
+ ID string
+ // Amount of time a transaction can remain unresolved (neither committed nor aborted)
+ // default is 1 min
+ Timeout time.Duration
+
+ Retry struct {
+ // The total number of times to retry sending a message (default 50).
+ // Similar to the `message.send.max.retries` setting of the JVM producer.
+ Max int
+ // How long to wait for the cluster to settle between retries
+ // (default 10ms). Similar to the `retry.backoff.ms` setting of the
+ // JVM producer.
+ Backoff time.Duration
+ // Called to compute backoff time dynamically. Useful for implementing
+ // more sophisticated backoff strategies. This takes precedence over
+ // `Backoff` if set.
+ BackoffFunc func(retries, maxRetries int) time.Duration
+ }
+ }
+
+ // Return specifies what channels will be populated. If they are set to true,
+ // you must read from the respective channels to prevent deadlock. If,
+ // however, this config is used to create a `SyncProducer`, both must be set
+ // to true and you shall not read from the channels since the producer does
+ // this internally.
+ Return struct {
+ // If enabled, successfully delivered messages will be returned on the
+ // Successes channel (default disabled).
+ Successes bool
+
+ // If enabled, messages that failed to deliver will be returned on the
+ // Errors channel, including error (default enabled).
+ Errors bool
+ }
+
+ // The following config options control how often messages are batched up and
+ // sent to the broker. By default, messages are sent as fast as possible, and
+ // all messages received while the current batch is in-flight are placed
+ // into the subsequent batch.
+ Flush struct {
+ // The best-effort number of bytes needed to trigger a flush. Use the
+ // global sarama.MaxRequestSize to set a hard upper limit.
+ Bytes int
+ // The best-effort number of messages needed to trigger a flush. Use
+ // `MaxMessages` to set a hard upper limit.
+ Messages int
+ // The best-effort frequency of flushes. Equivalent to
+ // `queue.buffering.max.ms` setting of JVM producer.
+ Frequency time.Duration
+ // The maximum number of messages the producer will send in a single
+ // broker request. Defaults to 0 for unlimited. Similar to
+ // `queue.buffering.max.messages` in the JVM producer.
+ MaxMessages int
+ }
+
+ Retry struct {
+ // The total number of times to retry sending a message (default 3).
+ // Similar to the `message.send.max.retries` setting of the JVM producer.
+ Max int
+ // How long to wait for the cluster to settle between retries
+ // (default 100ms). Similar to the `retry.backoff.ms` setting of the
+ // JVM producer.
+ Backoff time.Duration
+ // Called to compute backoff time dynamically. Useful for implementing
+ // more sophisticated backoff strategies. This takes precedence over
+ // `Backoff` if set.
+ BackoffFunc func(retries, maxRetries int) time.Duration
+ // The maximum length of the bridging buffer between `input` and `retries` channels
+ // in AsyncProducer#retryHandler.
+ // The limit is to prevent this buffer from overflowing or causing OOM.
+ // Defaults to 0 for unlimited.
+ // Any value between 0 and 4096 is pushed to 4096.
+ // A zero or negative value indicates unlimited.
+ MaxBufferLength int
+ // The maximum total byte size of messages in the bridging buffer between `input`
+ // and `retries` channels in AsyncProducer#retryHandler.
+ // This limit prevents the buffer from consuming excessive memory.
+ // Defaults to 0 for unlimited.
+ // Any value between 0 and 32 MB is pushed to 32 MB.
+ // A zero or negative value indicates unlimited.
+ MaxBufferBytes int64
+ }
+
+ // Interceptors to be called when the producer dispatcher reads the
+ // message for the first time. Interceptors allows to intercept and
+ // possible mutate the message before they are published to Kafka
+ // cluster. *ProducerMessage modified by the first interceptor's
+ // OnSend() is passed to the second interceptor OnSend(), and so on in
+ // the interceptor chain.
+ Interceptors []ProducerInterceptor
+ }
+
+ // Consumer is the namespace for configuration related to consuming messages,
+ // used by the Consumer.
+ Consumer struct {
+ // Group is the namespace for configuring consumer group.
+ Group struct {
+ Session struct {
+ // The timeout used to detect consumer failures when using Kafka's group management facility.
+ // The consumer sends periodic heartbeats to indicate its liveness to the broker.
+ // If no heartbeats are received by the broker before the expiration of this session timeout,
+ // then the broker will remove this consumer from the group and initiate a rebalance.
+ // Note that the value must be in the allowable range as configured in the broker configuration
+ // by `group.min.session.timeout.ms` and `group.max.session.timeout.ms` (default 10s)
+ Timeout time.Duration
+ }
+ Heartbeat struct {
+ // The expected time between heartbeats to the consumer coordinator when using Kafka's group
+ // management facilities. Heartbeats are used to ensure that the consumer's session stays active and
+ // to facilitate rebalancing when new consumers join or leave the group.
+ // The value must be set lower than Consumer.Group.Session.Timeout, but typically should be set no
+ // higher than 1/3 of that value.
+ // It can be adjusted even lower to control the expected time for normal rebalances (default 3s)
+ Interval time.Duration
+ }
+ Rebalance struct {
+ // Strategy for allocating topic partitions to members.
+ // Deprecated: Strategy exists for historical compatibility
+ // and should not be used. Please use GroupStrategies.
+ Strategy BalanceStrategy
+
+ // GroupStrategies is the priority-ordered list of client-side consumer group
+ // balancing strategies that will be offered to the coordinator. The first
+ // strategy that all group members support will be chosen by the leader.
+ // default: [ NewBalanceStrategyRange() ]
+ GroupStrategies []BalanceStrategy
+
+ // The maximum allowed time for each worker to join the group once a rebalance has begun.
+ // This is basically a limit on the amount of time needed for all tasks to flush any pending
+ // data and commit offsets. If the timeout is exceeded, then the worker will be removed from
+ // the group, which will cause offset commit failures (default 60s).
+ Timeout time.Duration
+
+ Retry struct {
+ // When a new consumer joins a consumer group the set of consumers attempt to "rebalance"
+ // the load to assign partitions to each consumer. If the set of consumers changes while
+ // this assignment is taking place the rebalance will fail and retry. This setting controls
+ // the maximum number of attempts before giving up (default 4).
+ Max int
+ // Backoff time between retries during rebalance (default 2s)
+ Backoff time.Duration
+ }
+ }
+ Member struct {
+ // Custom metadata to include when joining the group. The user data for all joined members
+ // can be retrieved by sending a DescribeGroupRequest to the broker that is the
+ // coordinator for the group.
+ UserData []byte
+ }
+
+ // support KIP-345
+ InstanceId string
+
+ // If true, consumer offsets will be automatically reset to configured Initial value
+ // if the fetched consumer offset is out of range of available offsets. Out of range
+ // can happen if the data has been deleted from the server, or during situations of
+ // under-replication where a replica does not have all the data yet. It can be
+ // dangerous to reset the offset automatically, particularly in the latter case. Defaults
+ // to true to maintain existing behavior.
+ ResetInvalidOffsets bool
+ }
+
+ Retry struct {
+ // How long to wait after a failing to read from a partition before
+ // trying again (default 2s).
+ Backoff time.Duration
+ // Called to compute backoff time dynamically. Useful for implementing
+ // more sophisticated backoff strategies. This takes precedence over
+ // `Backoff` if set.
+ BackoffFunc func(retries int) time.Duration
+ }
+
+ // Fetch is the namespace for controlling how many bytes are retrieved by any
+ // given request.
+ Fetch struct {
+ // The minimum number of message bytes to fetch in a request - the broker
+ // will wait until at least this many are available. The default is 1,
+ // as 0 causes the consumer to spin when no messages are available.
+ // Equivalent to the JVM's `fetch.min.bytes`.
+ Min int32
+ // The default number of message bytes to fetch from the broker in each
+ // request (default 1MB). This should be larger than the majority of
+ // your messages, or else the consumer will spend a lot of time
+ // negotiating sizes and not actually consuming. Similar to the JVM's
+ // `fetch.message.max.bytes`.
+ Default int32
+ // The maximum number of message bytes to fetch from the broker in a
+ // single request. Messages larger than this will return
+ // ErrMessageTooLarge and will not be consumable, so you must be sure
+ // this is at least as large as your largest message. Defaults to 0
+ // (no limit). Similar to the JVM's `fetch.message.max.bytes`. The
+ // global `sarama.MaxResponseSize` still applies.
+ Max int32
+ }
+ // The maximum amount of time the broker will wait for Consumer.Fetch.Min
+ // bytes to become available before it returns fewer than that anyways. The
+ // default is 250ms, since 0 causes the consumer to spin when no events are
+ // available. 100-500ms is a reasonable range for most cases. Kafka only
+ // supports precision up to milliseconds; nanoseconds will be truncated.
+ // Equivalent to the JVM's `fetch.max.wait.ms`.
+ MaxWaitTime time.Duration
+
+ // The maximum amount of time the consumer expects a message takes to
+ // process for the user. If writing to the Messages channel takes longer
+ // than this, that partition will stop fetching more messages until it
+ // can proceed again.
+ // Note that, since the Messages channel is buffered, the actual grace time is
+ // (MaxProcessingTime * ChannelBufferSize). Defaults to 100ms.
+ // If a message is not written to the Messages channel between two ticks
+ // of the expiryTicker then a timeout is detected.
+ // Using a ticker instead of a timer to detect timeouts should typically
+ // result in many fewer calls to Timer functions which may result in a
+ // significant performance improvement if many messages are being sent
+ // and timeouts are infrequent.
+ // The disadvantage of using a ticker instead of a timer is that
+ // timeouts will be less accurate. That is, the effective timeout could
+ // be between `MaxProcessingTime` and `2 * MaxProcessingTime`. For
+ // example, if `MaxProcessingTime` is 100ms then a delay of 180ms
+ // between two messages being sent may not be recognized as a timeout.
+ MaxProcessingTime time.Duration
+
+ // Return specifies what channels will be populated. If they are set to true,
+ // you must read from them to prevent deadlock.
+ Return struct {
+ // If enabled, any errors that occurred while consuming are returned on
+ // the Errors channel (default disabled).
+ Errors bool
+ }
+
+ // Offsets specifies configuration for how and when to commit consumed
+ // offsets. This currently requires the manual use of an OffsetManager
+ // but will eventually be automated.
+ Offsets struct {
+ // Deprecated: CommitInterval exists for historical compatibility
+ // and should not be used. Please use Consumer.Offsets.AutoCommit
+ CommitInterval time.Duration
+
+ // AutoCommit specifies configuration for commit messages automatically.
+ AutoCommit struct {
+ // Whether or not to auto-commit updated offsets back to the broker.
+ // (default enabled).
+ Enable bool
+
+ // How frequently to commit updated offsets. Ineffective unless
+ // auto-commit is enabled (default 1s)
+ Interval time.Duration
+ }
+
+ // The initial offset to use if no offset was previously committed.
+ // Should be OffsetNewest or OffsetOldest. Defaults to OffsetNewest.
+ Initial int64
+
+ // The retention duration for committed offsets. If zero, disabled
+ // (in which case the `offsets.retention.minutes` option on the
+ // broker will be used). Kafka only supports precision up to
+ // milliseconds; nanoseconds will be truncated. Requires Kafka
+ // broker version 0.9.0 or later.
+ // (default is 0: disabled).
+ Retention time.Duration
+
+ Retry struct {
+ // The total number of times to retry failing commit
+ // requests during OffsetManager shutdown (default 3).
+ Max int
+ }
+ }
+
+ // IsolationLevel support 2 mode:
+ // - use `ReadUncommitted` (default) to consume and return all messages in message channel
+ // - use `ReadCommitted` to hide messages that are part of an aborted transaction
+ IsolationLevel IsolationLevel
+
+ // Interceptors to be called just before the record is sent to the
+ // messages channel. Interceptors allows to intercept and possible
+ // mutate the message before they are returned to the client.
+ // *ConsumerMessage modified by the first interceptor's OnConsume() is
+ // passed to the second interceptor OnConsume(), and so on in the
+ // interceptor chain.
+ Interceptors []ConsumerInterceptor
+ }
+
+ // A user-provided string sent with every request to the brokers for logging,
+ // debugging, and auditing purposes. Defaults to "sarama", but you should
+ // probably set it to something specific to your application.
+ ClientID string
+ // A rack identifier for this client. This can be any string value which
+ // indicates where this client is physically located.
+ // It corresponds with the broker config 'broker.rack'
+ RackID string
+ // The number of events to buffer in internal and external channels. This
+ // permits the producer and consumer to continue processing some messages
+ // in the background while user code is working, greatly improving throughput.
+ // Defaults to 256.
+ ChannelBufferSize int
+ // ApiVersionsRequest determines whether Sarama should send an
+ // ApiVersionsRequest message to each broker as part of its initial
+ // connection. This defaults to `true` to match the official Java client
+ // and most 3rdparty ones.
+ ApiVersionsRequest bool
+ // The version of Kafka that Sarama will assume it is running against.
+ // Defaults to the oldest supported stable version. Since Kafka provides
+ // backwards-compatibility, setting it to a version older than you have
+ // will not break anything, although it may prevent you from using the
+ // latest features. Setting it to a version greater than you are actually
+ // running may lead to random breakage.
+ Version KafkaVersion
+ // The registry to define metrics into.
+ // Defaults to a local registry.
+ // If you want to disable metrics gathering, set "metrics.UseNilMetrics" to "true"
+ // prior to starting Sarama.
+ // See Examples on how to use the metrics registry
+ MetricRegistry metrics.Registry
+}
+
+// NewConfig returns a new configuration instance with sane defaults.
+func NewConfig() *Config {
+ c := &Config{}
+
+ c.Admin.Retry.Max = 5
+ c.Admin.Retry.Backoff = 100 * time.Millisecond
+ c.Admin.Timeout = 3 * time.Second
+
+ c.Net.MaxOpenRequests = 5
+ c.Net.DialTimeout = 30 * time.Second
+ c.Net.ReadTimeout = 30 * time.Second
+ c.Net.WriteTimeout = 30 * time.Second
+ c.Net.SASL.Handshake = true
+ c.Net.SASL.Version = SASLHandshakeV1
+
+ c.Metadata.Retry.Max = 3
+ c.Metadata.Retry.Backoff = 250 * time.Millisecond
+ c.Metadata.RefreshFrequency = 10 * time.Minute
+ c.Metadata.Full = true
+ c.Metadata.AllowAutoTopicCreation = true
+ c.Metadata.SingleFlight = true
+
+ c.Producer.MaxMessageBytes = 1024 * 1024
+ c.Producer.RequiredAcks = WaitForLocal
+ c.Producer.Timeout = 10 * time.Second
+ c.Producer.Partitioner = NewHashPartitioner
+ c.Producer.Retry.Max = 3
+ c.Producer.Retry.Backoff = 100 * time.Millisecond
+ c.Producer.Return.Errors = true
+ c.Producer.CompressionLevel = CompressionLevelDefault
+
+ c.Producer.Transaction.Timeout = 1 * time.Minute
+ c.Producer.Transaction.Retry.Max = 50
+ c.Producer.Transaction.Retry.Backoff = 100 * time.Millisecond
+
+ c.Consumer.Fetch.Min = 1
+ c.Consumer.Fetch.Default = 1024 * 1024
+ c.Consumer.Retry.Backoff = 2 * time.Second
+ c.Consumer.MaxWaitTime = 500 * time.Millisecond
+ c.Consumer.MaxProcessingTime = 100 * time.Millisecond
+ c.Consumer.Return.Errors = false
+ c.Consumer.Offsets.AutoCommit.Enable = true
+ c.Consumer.Offsets.AutoCommit.Interval = 1 * time.Second
+ c.Consumer.Offsets.Initial = OffsetNewest
+ c.Consumer.Offsets.Retry.Max = 3
+
+ c.Consumer.Group.Session.Timeout = 10 * time.Second
+ c.Consumer.Group.Heartbeat.Interval = 3 * time.Second
+ c.Consumer.Group.Rebalance.GroupStrategies = []BalanceStrategy{NewBalanceStrategyRange()}
+ c.Consumer.Group.Rebalance.Timeout = 60 * time.Second
+ c.Consumer.Group.Rebalance.Retry.Max = 4
+ c.Consumer.Group.Rebalance.Retry.Backoff = 2 * time.Second
+ c.Consumer.Group.ResetInvalidOffsets = true
+
+ c.ClientID = defaultClientID
+ c.ChannelBufferSize = 256
+ c.ApiVersionsRequest = true
+ c.Version = DefaultVersion
+ c.MetricRegistry = metrics.NewRegistry()
+
+ return c
+}
+
+// Validate checks a Config instance. It will return a
+// ConfigurationError if the specified values don't make sense.
+//
+//nolint:gocyclo // This function's cyclomatic complexity has go beyond 100
+func (c *Config) Validate() error {
+ // some configuration values should be warned on but not fail completely, do those first
+ if !c.Net.TLS.Enable && c.Net.TLS.Config != nil {
+ Logger.Println("Net.TLS is disabled but a non-nil configuration was provided.")
+ }
+ if !c.Net.SASL.Enable {
+ if c.Net.SASL.User != "" {
+ Logger.Println("Net.SASL is disabled but a non-empty username was provided.")
+ }
+ if c.Net.SASL.Password != "" {
+ Logger.Println("Net.SASL is disabled but a non-empty password was provided.")
+ }
+ }
+ if c.Producer.RequiredAcks > 1 {
+ Logger.Println("Producer.RequiredAcks > 1 is deprecated and will raise an exception with kafka >= 0.8.2.0.")
+ }
+ if c.Producer.MaxMessageBytes >= int(MaxRequestSize) {
+ Logger.Println("Producer.MaxMessageBytes must be smaller than MaxRequestSize; it will be ignored.")
+ }
+ if c.Producer.Flush.Bytes >= int(MaxRequestSize) {
+ Logger.Println("Producer.Flush.Bytes must be smaller than MaxRequestSize; it will be ignored.")
+ }
+ if (c.Producer.Flush.Bytes > 0 || c.Producer.Flush.Messages > 0) && c.Producer.Flush.Frequency == 0 {
+ Logger.Println("Producer.Flush: Bytes or Messages are set, but Frequency is not; messages may not get flushed.")
+ }
+ if c.Producer.Timeout%time.Millisecond != 0 {
+ Logger.Println("Producer.Timeout only supports millisecond resolution; nanoseconds will be truncated.")
+ }
+ if c.Consumer.MaxWaitTime < 100*time.Millisecond {
+ Logger.Println("Consumer.MaxWaitTime is very low, which can cause high CPU and network usage. See documentation for details.")
+ }
+ if c.Consumer.MaxWaitTime%time.Millisecond != 0 {
+ Logger.Println("Consumer.MaxWaitTime only supports millisecond precision; nanoseconds will be truncated.")
+ }
+ if c.Consumer.Offsets.Retention%time.Millisecond != 0 {
+ Logger.Println("Consumer.Offsets.Retention only supports millisecond precision; nanoseconds will be truncated.")
+ }
+ if c.Consumer.Group.Session.Timeout%time.Millisecond != 0 {
+ Logger.Println("Consumer.Group.Session.Timeout only supports millisecond precision; nanoseconds will be truncated.")
+ }
+ if c.Consumer.Group.Heartbeat.Interval%time.Millisecond != 0 {
+ Logger.Println("Consumer.Group.Heartbeat.Interval only supports millisecond precision; nanoseconds will be truncated.")
+ }
+ if c.Consumer.Group.Rebalance.Timeout%time.Millisecond != 0 {
+ Logger.Println("Consumer.Group.Rebalance.Timeout only supports millisecond precision; nanoseconds will be truncated.")
+ }
+ if c.ClientID == defaultClientID {
+ Logger.Println("ClientID is the default of 'sarama', you should consider setting it to something application-specific.")
+ }
+
+ // validate Net values
+ switch {
+ case c.Net.MaxOpenRequests <= 0:
+ return ConfigurationError("Net.MaxOpenRequests must be > 0")
+ case c.Net.DialTimeout <= 0:
+ return ConfigurationError("Net.DialTimeout must be > 0")
+ case c.Net.ReadTimeout <= 0:
+ return ConfigurationError("Net.ReadTimeout must be > 0")
+ case c.Net.WriteTimeout <= 0:
+ return ConfigurationError("Net.WriteTimeout must be > 0")
+ case c.Net.SASL.Enable:
+ if c.Net.SASL.Mechanism == "" {
+ c.Net.SASL.Mechanism = SASLTypePlaintext
+ }
+ if c.Net.SASL.Version == SASLHandshakeV0 && c.ApiVersionsRequest {
+ return ConfigurationError("ApiVersionsRequest must be disabled when SASL v0 is enabled")
+ }
+ switch c.Net.SASL.Mechanism {
+ case SASLTypePlaintext:
+ if c.Net.SASL.User == "" {
+ return ConfigurationError("Net.SASL.User must not be empty when SASL is enabled")
+ }
+ if c.Net.SASL.Password == "" {
+ return ConfigurationError("Net.SASL.Password must not be empty when SASL is enabled")
+ }
+ case SASLTypeOAuth:
+ if c.Net.SASL.TokenProvider == nil {
+ return ConfigurationError("An AccessTokenProvider instance must be provided to Net.SASL.TokenProvider")
+ }
+ case SASLTypeSCRAMSHA256, SASLTypeSCRAMSHA512:
+ if c.Net.SASL.User == "" {
+ return ConfigurationError("Net.SASL.User must not be empty when SASL is enabled")
+ }
+ if c.Net.SASL.Password == "" {
+ return ConfigurationError("Net.SASL.Password must not be empty when SASL is enabled")
+ }
+ if c.Net.SASL.SCRAMClientGeneratorFunc == nil {
+ return ConfigurationError("A SCRAMClientGeneratorFunc function must be provided to Net.SASL.SCRAMClientGeneratorFunc")
+ }
+ case SASLTypeGSSAPI:
+ if c.Net.SASL.GSSAPI.ServiceName == "" {
+ return ConfigurationError("Net.SASL.GSSAPI.ServiceName must not be empty when GSS-API mechanism is used")
+ }
+
+ switch c.Net.SASL.GSSAPI.AuthType {
+ case KRB5_USER_AUTH:
+ if c.Net.SASL.GSSAPI.Password == "" {
+ return ConfigurationError("Net.SASL.GSSAPI.Password must not be empty when GSS-API " +
+ "mechanism is used and Net.SASL.GSSAPI.AuthType = KRB5_USER_AUTH")
+ }
+ case KRB5_KEYTAB_AUTH:
+ if c.Net.SASL.GSSAPI.KeyTabPath == "" {
+ return ConfigurationError("Net.SASL.GSSAPI.KeyTabPath must not be empty when GSS-API mechanism is used" +
+ " and Net.SASL.GSSAPI.AuthType = KRB5_KEYTAB_AUTH")
+ }
+ case KRB5_CCACHE_AUTH:
+ if c.Net.SASL.GSSAPI.CCachePath == "" {
+ return ConfigurationError("Net.SASL.GSSAPI.CCachePath must not be empty when GSS-API mechanism is used" +
+ " and Net.SASL.GSSAPI.AuthType = KRB5_CCACHE_AUTH")
+ }
+ default:
+ return ConfigurationError("Net.SASL.GSSAPI.AuthType is invalid. Possible values are KRB5_USER_AUTH, KRB5_KEYTAB_AUTH, and KRB5_CCACHE_AUTH")
+ }
+
+ if c.Net.SASL.GSSAPI.KerberosConfigPath == "" {
+ return ConfigurationError("Net.SASL.GSSAPI.KerberosConfigPath must not be empty when GSS-API mechanism is used")
+ }
+ if c.Net.SASL.GSSAPI.Username == "" {
+ return ConfigurationError("Net.SASL.GSSAPI.Username must not be empty when GSS-API mechanism is used")
+ }
+ if c.Net.SASL.GSSAPI.Realm == "" {
+ return ConfigurationError("Net.SASL.GSSAPI.Realm must not be empty when GSS-API mechanism is used")
+ }
+ default:
+ msg := fmt.Sprintf("The SASL mechanism configuration is invalid. Possible values are `%s`, `%s`, `%s`, `%s` and `%s`",
+ SASLTypeOAuth, SASLTypePlaintext, SASLTypeSCRAMSHA256, SASLTypeSCRAMSHA512, SASLTypeGSSAPI)
+ return ConfigurationError(msg)
+ }
+ }
+
+ // validate the Admin values
+ switch {
+ case c.Admin.Timeout <= 0:
+ return ConfigurationError("Admin.Timeout must be > 0")
+ }
+
+ // validate the Metadata values
+ switch {
+ case c.Metadata.Retry.Max < 0:
+ return ConfigurationError("Metadata.Retry.Max must be >= 0")
+ case c.Metadata.Retry.Backoff < 0:
+ return ConfigurationError("Metadata.Retry.Backoff must be >= 0")
+ case c.Metadata.RefreshFrequency < 0:
+ return ConfigurationError("Metadata.RefreshFrequency must be >= 0")
+ }
+
+ // validate the Producer values
+ switch {
+ case c.Producer.MaxMessageBytes <= 0:
+ return ConfigurationError("Producer.MaxMessageBytes must be > 0")
+ case c.Producer.RequiredAcks < -1:
+ return ConfigurationError("Producer.RequiredAcks must be >= -1")
+ case c.Producer.Timeout <= 0:
+ return ConfigurationError("Producer.Timeout must be > 0")
+ case c.Producer.Partitioner == nil:
+ return ConfigurationError("Producer.Partitioner must not be nil")
+ case c.Producer.Flush.Bytes < 0:
+ return ConfigurationError("Producer.Flush.Bytes must be >= 0")
+ case c.Producer.Flush.Messages < 0:
+ return ConfigurationError("Producer.Flush.Messages must be >= 0")
+ case c.Producer.Flush.Frequency < 0:
+ return ConfigurationError("Producer.Flush.Frequency must be >= 0")
+ case c.Producer.Flush.MaxMessages < 0:
+ return ConfigurationError("Producer.Flush.MaxMessages must be >= 0")
+ case c.Producer.Flush.MaxMessages > 0 && c.Producer.Flush.MaxMessages < c.Producer.Flush.Messages:
+ return ConfigurationError("Producer.Flush.MaxMessages must be >= Producer.Flush.Messages when set")
+ case c.Producer.Retry.Max < 0:
+ return ConfigurationError("Producer.Retry.Max must be >= 0")
+ case c.Producer.Retry.Backoff < 0:
+ return ConfigurationError("Producer.Retry.Backoff must be >= 0")
+ }
+
+ if c.Producer.Compression == CompressionLZ4 && !c.Version.IsAtLeast(V0_10_0_0) {
+ return ConfigurationError("lz4 compression requires Version >= V0_10_0_0")
+ }
+
+ if c.Producer.Compression == CompressionGZIP {
+ if c.Producer.CompressionLevel != CompressionLevelDefault {
+ if _, err := gzip.NewWriterLevel(io.Discard, c.Producer.CompressionLevel); err != nil {
+ return ConfigurationError(fmt.Sprintf("gzip compression does not work with level %d: %v", c.Producer.CompressionLevel, err))
+ }
+ }
+ }
+
+ if c.Producer.Compression == CompressionZSTD && !c.Version.IsAtLeast(V2_1_0_0) {
+ return ConfigurationError("zstd compression requires Version >= V2_1_0_0")
+ }
+
+ if c.Producer.Idempotent {
+ if !c.Version.IsAtLeast(V0_11_0_0) {
+ return ConfigurationError("Idempotent producer requires Version >= V0_11_0_0")
+ }
+ if c.Producer.Retry.Max == 0 {
+ return ConfigurationError("Idempotent producer requires Producer.Retry.Max >= 1")
+ }
+ if c.Producer.RequiredAcks != WaitForAll {
+ return ConfigurationError("Idempotent producer requires Producer.RequiredAcks to be WaitForAll")
+ }
+ if c.Net.MaxOpenRequests > 1 {
+ return ConfigurationError("Idempotent producer requires Net.MaxOpenRequests to be 1")
+ }
+ }
+
+ if c.Producer.Transaction.ID != "" && !c.Producer.Idempotent {
+ return ConfigurationError("Transactional producer requires Idempotent to be true")
+ }
+
+ // validate the Consumer values
+ switch {
+ case c.Consumer.Fetch.Min <= 0:
+ return ConfigurationError("Consumer.Fetch.Min must be > 0")
+ case c.Consumer.Fetch.Default <= 0:
+ return ConfigurationError("Consumer.Fetch.Default must be > 0")
+ case c.Consumer.Fetch.Max < 0:
+ return ConfigurationError("Consumer.Fetch.Max must be >= 0")
+ case c.Consumer.MaxWaitTime < 1*time.Millisecond:
+ return ConfigurationError("Consumer.MaxWaitTime must be >= 1ms")
+ case c.Consumer.MaxProcessingTime <= 0:
+ return ConfigurationError("Consumer.MaxProcessingTime must be > 0")
+ case c.Consumer.Retry.Backoff < 0:
+ return ConfigurationError("Consumer.Retry.Backoff must be >= 0")
+ case c.Consumer.Offsets.AutoCommit.Interval <= 0:
+ return ConfigurationError("Consumer.Offsets.AutoCommit.Interval must be > 0")
+ case c.Consumer.Offsets.Initial != OffsetOldest && c.Consumer.Offsets.Initial != OffsetNewest:
+ return ConfigurationError("Consumer.Offsets.Initial must be OffsetOldest or OffsetNewest")
+ case c.Consumer.Offsets.Retry.Max < 0:
+ return ConfigurationError("Consumer.Offsets.Retry.Max must be >= 0")
+ case c.Consumer.IsolationLevel != ReadUncommitted && c.Consumer.IsolationLevel != ReadCommitted:
+ return ConfigurationError("Consumer.IsolationLevel must be ReadUncommitted or ReadCommitted")
+ }
+
+ if c.Consumer.Offsets.CommitInterval != 0 {
+ Logger.Println("Deprecation warning: Consumer.Offsets.CommitInterval exists for historical compatibility" +
+ " and should not be used. Please use Consumer.Offsets.AutoCommit, the current value will be ignored")
+ }
+ if c.Consumer.Group.Rebalance.Strategy != nil {
+ Logger.Println("Deprecation warning: Consumer.Group.Rebalance.Strategy exists for historical compatibility" +
+ " and should not be used. Please use Consumer.Group.Rebalance.GroupStrategies")
+ }
+
+ // validate IsolationLevel
+ if c.Consumer.IsolationLevel == ReadCommitted && !c.Version.IsAtLeast(V0_11_0_0) {
+ return ConfigurationError("ReadCommitted requires Version >= V0_11_0_0")
+ }
+
+ // validate the Consumer Group values
+ switch {
+ case c.Consumer.Group.Session.Timeout <= 2*time.Millisecond:
+ return ConfigurationError("Consumer.Group.Session.Timeout must be >= 2ms")
+ case c.Consumer.Group.Heartbeat.Interval < 1*time.Millisecond:
+ return ConfigurationError("Consumer.Group.Heartbeat.Interval must be >= 1ms")
+ case c.Consumer.Group.Heartbeat.Interval >= c.Consumer.Group.Session.Timeout:
+ return ConfigurationError("Consumer.Group.Heartbeat.Interval must be < Consumer.Group.Session.Timeout")
+ case c.Consumer.Group.Rebalance.Strategy == nil && len(c.Consumer.Group.Rebalance.GroupStrategies) == 0:
+ return ConfigurationError("Consumer.Group.Rebalance.GroupStrategies or Consumer.Group.Rebalance.Strategy must not be empty")
+ case c.Consumer.Group.Rebalance.Timeout <= time.Millisecond:
+ return ConfigurationError("Consumer.Group.Rebalance.Timeout must be >= 1ms")
+ case c.Consumer.Group.Rebalance.Retry.Max < 0:
+ return ConfigurationError("Consumer.Group.Rebalance.Retry.Max must be >= 0")
+ case c.Consumer.Group.Rebalance.Retry.Backoff < 0:
+ return ConfigurationError("Consumer.Group.Rebalance.Retry.Backoff must be >= 0")
+ }
+
+ for _, strategy := range c.Consumer.Group.Rebalance.GroupStrategies {
+ if strategy == nil {
+ return ConfigurationError("elements in Consumer.Group.Rebalance.Strategies must not be empty")
+ }
+ }
+
+ if c.Consumer.Group.InstanceId != "" {
+ if !c.Version.IsAtLeast(V2_3_0_0) {
+ return ConfigurationError("Consumer.Group.InstanceId need Version >= 2.3")
+ }
+ if err := validateGroupInstanceId(c.Consumer.Group.InstanceId); err != nil {
+ return err
+ }
+ }
+
+ // validate misc shared values
+ switch {
+ case c.ChannelBufferSize < 0:
+ return ConfigurationError("ChannelBufferSize must be >= 0")
+ }
+
+ // only validate clientID locally for Kafka versions before KIP-190 was implemented
+ if !c.Version.IsAtLeast(V1_0_0_0) && !validClientID.MatchString(c.ClientID) {
+ return ConfigurationError(fmt.Sprintf("ClientID value %q is not valid for Kafka versions before 1.0.0", c.ClientID))
+ }
+
+ return nil
+}
+
+func (c *Config) getDialer() proxy.Dialer {
+ if c.Net.Proxy.Enable {
+ Logger.Println("using proxy")
+ return c.Net.Proxy.Dialer
+ } else {
+ return &net.Dialer{
+ Timeout: c.Net.DialTimeout,
+ KeepAlive: c.Net.KeepAlive,
+ LocalAddr: c.Net.LocalAddr,
+ }
+ }
+}
+
+const MAX_GROUP_INSTANCE_ID_LENGTH = 249
+
+var GROUP_INSTANCE_ID_REGEXP = regexp.MustCompile(`^[0-9a-zA-Z\._\-]+$`)
+
+func validateGroupInstanceId(id string) error {
+ if id == "" {
+ return ConfigurationError("Group instance id must be non-empty string")
+ }
+ if id == "." || id == ".." {
+ return ConfigurationError(`Group instance id cannot be "." or ".."`)
+ }
+ if len(id) > MAX_GROUP_INSTANCE_ID_LENGTH {
+ return ConfigurationError(fmt.Sprintf(`Group instance id cannot be longer than %v, characters: %s`, MAX_GROUP_INSTANCE_ID_LENGTH, id))
+ }
+ if !GROUP_INSTANCE_ID_REGEXP.MatchString(id) {
+ return ConfigurationError(fmt.Sprintf(`Group instance id %s is illegal, it contains a character other than, '.', '_' and '-'`, id))
+ }
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/config_resource_type.go b/vendor/github.com/IBM/sarama/config_resource_type.go
new file mode 100644
index 0000000..bef1053
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/config_resource_type.go
@@ -0,0 +1,18 @@
+package sarama
+
+// ConfigResourceType is a type for resources that have configs.
+type ConfigResourceType int8
+
+// Taken from:
+// https://github.com/apache/kafka/blob/ed7c071e07f1f90e4c2895582f61ca090ced3c42/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java#L32-L55
+
+const (
+ // UnknownResource constant type
+ UnknownResource ConfigResourceType = 0
+ // TopicResource constant type
+ TopicResource ConfigResourceType = 2
+ // BrokerResource constant type
+ BrokerResource ConfigResourceType = 4
+ // BrokerLoggerResource constant type
+ BrokerLoggerResource ConfigResourceType = 8
+)
diff --git a/vendor/github.com/IBM/sarama/consumer.go b/vendor/github.com/IBM/sarama/consumer.go
new file mode 100644
index 0000000..4c96af4
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/consumer.go
@@ -0,0 +1,1136 @@
+package sarama
+
+import (
+ "errors"
+ "fmt"
+ "math"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/rcrowley/go-metrics"
+)
+
+// ConsumerMessage encapsulates a Kafka message returned by the consumer.
+type ConsumerMessage struct {
+ Headers []*RecordHeader // only set if kafka is version 0.11+
+ Timestamp time.Time // only set if kafka is version 0.10+, inner message timestamp
+ BlockTimestamp time.Time // only set if kafka is version 0.10+, outer (compressed) block timestamp
+
+ Key, Value []byte
+ Topic string
+ Partition int32
+ Offset int64
+}
+
+// ConsumerError is what is provided to the user when an error occurs.
+// It wraps an error and includes the topic and partition.
+type ConsumerError struct {
+ Topic string
+ Partition int32
+ Err error
+}
+
+func (ce ConsumerError) Error() string {
+ return fmt.Sprintf("kafka: error while consuming %s/%d: %s", ce.Topic, ce.Partition, ce.Err)
+}
+
+func (ce ConsumerError) Unwrap() error {
+ return ce.Err
+}
+
+// ConsumerErrors is a type that wraps a batch of errors and implements the Error interface.
+// It can be returned from the PartitionConsumer's Close methods to avoid the need to manually drain errors
+// when stopping.
+type ConsumerErrors []*ConsumerError
+
+func (ce ConsumerErrors) Error() string {
+ return fmt.Sprintf("kafka: %d errors while consuming", len(ce))
+}
+
+// Consumer manages PartitionConsumers which process Kafka messages from brokers. You MUST call Close()
+// on a consumer to avoid leaks, it will not be garbage-collected automatically when it passes out of
+// scope.
+type Consumer interface {
+ // Topics returns the set of available topics as retrieved from the cluster
+ // metadata. This method is the same as Client.Topics(), and is provided for
+ // convenience.
+ Topics() ([]string, error)
+
+ // Partitions returns the sorted list of all partition IDs for the given topic.
+ // This method is the same as Client.Partitions(), and is provided for convenience.
+ Partitions(topic string) ([]int32, error)
+
+ // ConsumePartition creates a PartitionConsumer on the given topic/partition with
+ // the given offset. It will return an error if this Consumer is already consuming
+ // on the given topic/partition. Offset can be a literal offset, or OffsetNewest
+ // or OffsetOldest
+ ConsumePartition(topic string, partition int32, offset int64) (PartitionConsumer, error)
+
+ // HighWaterMarks returns the current high water marks for each topic and partition.
+ // Consistency between partitions is not guaranteed since high water marks are updated separately.
+ HighWaterMarks() map[string]map[int32]int64
+
+ // Close shuts down the consumer. It must be called after all child
+ // PartitionConsumers have already been closed.
+ Close() error
+
+ // Pause suspends fetching from the requested partitions. Future calls to the broker will not return any
+ // records from these partitions until they have been resumed using Resume()/ResumeAll().
+ // Note that this method does not affect partition subscription.
+ // In particular, it does not cause a group rebalance when automatic assignment is used.
+ Pause(topicPartitions map[string][]int32)
+
+ // Resume resumes specified partitions which have been paused with Pause()/PauseAll().
+ // New calls to the broker will return records from these partitions if there are any to be fetched.
+ Resume(topicPartitions map[string][]int32)
+
+ // PauseAll suspends fetching from all partitions. Future calls to the broker will not return any
+ // records from these partitions until they have been resumed using Resume()/ResumeAll().
+ // Note that this method does not affect partition subscription.
+ // In particular, it does not cause a group rebalance when automatic assignment is used.
+ PauseAll()
+
+ // ResumeAll resumes all partitions which have been paused with Pause()/PauseAll().
+ // New calls to the broker will return records from these partitions if there are any to be fetched.
+ ResumeAll()
+}
+
+// max time to wait for more partition subscriptions
+const partitionConsumersBatchTimeout = 100 * time.Millisecond
+
+type consumer struct {
+ conf *Config
+ children map[string]map[int32]*partitionConsumer
+ brokerConsumers map[*Broker]*brokerConsumer
+ client Client
+ metricRegistry metrics.Registry
+ lock sync.Mutex
+}
+
+// NewConsumer creates a new consumer using the given broker addresses and configuration.
+func NewConsumer(addrs []string, config *Config) (Consumer, error) {
+ client, err := NewClient(addrs, config)
+ if err != nil {
+ return nil, err
+ }
+ return newConsumer(client)
+}
+
+// NewConsumerFromClient creates a new consumer using the given client. It is still
+// necessary to call Close() on the underlying client when shutting down this consumer.
+func NewConsumerFromClient(client Client) (Consumer, error) {
+ // For clients passed in by the client, ensure we don't
+ // call Close() on it.
+ cli := &nopCloserClient{client}
+ return newConsumer(cli)
+}
+
+func newConsumer(client Client) (Consumer, error) {
+ // Check that we are not dealing with a closed Client before processing any other arguments
+ if client.Closed() {
+ return nil, ErrClosedClient
+ }
+
+ c := &consumer{
+ client: client,
+ conf: client.Config(),
+ children: make(map[string]map[int32]*partitionConsumer),
+ brokerConsumers: make(map[*Broker]*brokerConsumer),
+ metricRegistry: newCleanupRegistry(client.Config().MetricRegistry),
+ }
+
+ return c, nil
+}
+
+func (c *consumer) Close() error {
+ c.metricRegistry.UnregisterAll()
+ return c.client.Close()
+}
+
+func (c *consumer) Topics() ([]string, error) {
+ return c.client.Topics()
+}
+
+func (c *consumer) Partitions(topic string) ([]int32, error) {
+ return c.client.Partitions(topic)
+}
+
+func (c *consumer) ConsumePartition(topic string, partition int32, offset int64) (PartitionConsumer, error) {
+ child := &partitionConsumer{
+ consumer: c,
+ conf: c.conf,
+ topic: topic,
+ partition: partition,
+ messages: make(chan *ConsumerMessage, c.conf.ChannelBufferSize),
+ errors: make(chan *ConsumerError, c.conf.ChannelBufferSize),
+ feeder: make(chan *FetchResponse, 1),
+ leaderEpoch: invalidLeaderEpoch,
+ preferredReadReplica: invalidPreferredReplicaID,
+ trigger: make(chan none, 1),
+ dying: make(chan none),
+ fetchSize: c.conf.Consumer.Fetch.Default,
+ }
+
+ if err := child.chooseStartingOffset(offset); err != nil {
+ return nil, err
+ }
+
+ leader, epoch, err := c.client.LeaderAndEpoch(child.topic, child.partition)
+ if err != nil {
+ return nil, err
+ }
+
+ if err := c.addChild(child); err != nil {
+ return nil, err
+ }
+
+ go withRecover(child.dispatcher)
+ go withRecover(child.responseFeeder)
+
+ child.leaderEpoch = epoch
+ child.broker = c.refBrokerConsumer(leader)
+ child.broker.input <- child
+
+ return child, nil
+}
+
+func (c *consumer) HighWaterMarks() map[string]map[int32]int64 {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ hwms := make(map[string]map[int32]int64)
+ for topic, p := range c.children {
+ hwm := make(map[int32]int64, len(p))
+ for partition, pc := range p {
+ hwm[partition] = pc.HighWaterMarkOffset()
+ }
+ hwms[topic] = hwm
+ }
+
+ return hwms
+}
+
+func (c *consumer) addChild(child *partitionConsumer) error {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ topicChildren := c.children[child.topic]
+ if topicChildren == nil {
+ topicChildren = make(map[int32]*partitionConsumer)
+ c.children[child.topic] = topicChildren
+ }
+
+ if topicChildren[child.partition] != nil {
+ return ConfigurationError("That topic/partition is already being consumed")
+ }
+
+ topicChildren[child.partition] = child
+ return nil
+}
+
+func (c *consumer) removeChild(child *partitionConsumer) {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ delete(c.children[child.topic], child.partition)
+}
+
+func (c *consumer) refBrokerConsumer(broker *Broker) *brokerConsumer {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ bc := c.brokerConsumers[broker]
+ if bc == nil {
+ bc = c.newBrokerConsumer(broker)
+ c.brokerConsumers[broker] = bc
+ }
+
+ bc.refs++
+
+ return bc
+}
+
+func (c *consumer) unrefBrokerConsumer(brokerWorker *brokerConsumer) {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ brokerWorker.refs--
+
+ if brokerWorker.refs == 0 {
+ close(brokerWorker.input)
+ if c.brokerConsumers[brokerWorker.broker] == brokerWorker {
+ delete(c.brokerConsumers, brokerWorker.broker)
+ }
+ }
+}
+
+func (c *consumer) abandonBrokerConsumer(brokerWorker *brokerConsumer) {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ delete(c.brokerConsumers, brokerWorker.broker)
+}
+
+// Pause implements Consumer.
+func (c *consumer) Pause(topicPartitions map[string][]int32) {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ for topic, partitions := range topicPartitions {
+ for _, partition := range partitions {
+ if topicConsumers, ok := c.children[topic]; ok {
+ if partitionConsumer, ok := topicConsumers[partition]; ok {
+ partitionConsumer.Pause()
+ }
+ }
+ }
+ }
+}
+
+// Resume implements Consumer.
+func (c *consumer) Resume(topicPartitions map[string][]int32) {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ for topic, partitions := range topicPartitions {
+ for _, partition := range partitions {
+ if topicConsumers, ok := c.children[topic]; ok {
+ if partitionConsumer, ok := topicConsumers[partition]; ok {
+ partitionConsumer.Resume()
+ }
+ }
+ }
+ }
+}
+
+// PauseAll implements Consumer.
+func (c *consumer) PauseAll() {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ for _, partitions := range c.children {
+ for _, partitionConsumer := range partitions {
+ partitionConsumer.Pause()
+ }
+ }
+}
+
+// ResumeAll implements Consumer.
+func (c *consumer) ResumeAll() {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ for _, partitions := range c.children {
+ for _, partitionConsumer := range partitions {
+ partitionConsumer.Resume()
+ }
+ }
+}
+
+// PartitionConsumer
+
+// PartitionConsumer processes Kafka messages from a given topic and partition. You MUST call one of Close() or
+// AsyncClose() on a PartitionConsumer to avoid leaks; it will not be garbage-collected automatically when it passes out
+// of scope.
+//
+// The simplest way of using a PartitionConsumer is to loop over its Messages channel using a for/range
+// loop. The PartitionConsumer will only stop itself in one case: when the offset being consumed is reported
+// as out of range by the brokers. In this case you should decide what you want to do (try a different offset,
+// notify a human, etc) and handle it appropriately. For all other error cases, it will just keep retrying.
+// By default, it logs these errors to sarama.Logger; if you want to be notified directly of all errors, set
+// your config's Consumer.Return.Errors to true and read from the Errors channel, using a select statement
+// or a separate goroutine. Check out the Consumer examples to see implementations of these different approaches.
+//
+// To terminate such a for/range loop while the loop is executing, call AsyncClose. This will kick off the process of
+// consumer tear-down & return immediately. Continue to loop, servicing the Messages channel until the teardown process
+// AsyncClose initiated closes it (thus terminating the for/range loop). If you've already ceased reading Messages, call
+// Close; this will signal the PartitionConsumer's goroutines to begin shutting down (just like AsyncClose), but will
+// also drain the Messages channel, harvest all errors & return them once cleanup has completed.
+type PartitionConsumer interface {
+ // AsyncClose initiates a shutdown of the PartitionConsumer. This method will return immediately, after which you
+ // should continue to service the 'Messages' and 'Errors' channels until they are empty. It is required to call this
+ // function, or Close before a consumer object passes out of scope, as it will otherwise leak memory. You must call
+ // this before calling Close on the underlying client.
+ AsyncClose()
+
+ // Close stops the PartitionConsumer from fetching messages. It will initiate a shutdown just like AsyncClose, drain
+ // the Messages channel, harvest any errors & return them to the caller. Note that if you are continuing to service
+ // the Messages channel when this function is called, you will be competing with Close for messages; consider
+ // calling AsyncClose, instead. It is required to call this function (or AsyncClose) before a consumer object passes
+ // out of scope, as it will otherwise leak memory. You must call this before calling Close on the underlying client.
+ Close() error
+
+ // Messages returns the read channel for the messages that are returned by
+ // the broker.
+ Messages() <-chan *ConsumerMessage
+
+ // Errors returns a read channel of errors that occurred during consuming, if
+ // enabled. By default, errors are logged and not returned over this channel.
+ // If you want to implement any custom error handling, set your config's
+ // Consumer.Return.Errors setting to true, and read from this channel.
+ Errors() <-chan *ConsumerError
+
+ // HighWaterMarkOffset returns the high water mark offset of the partition,
+ // i.e. the offset that will be used for the next message that will be produced.
+ // You can use this to determine how far behind the processing is.
+ HighWaterMarkOffset() int64
+
+ // Pause suspends fetching from this partition. Future calls to the broker will not return
+ // any records from these partition until it have been resumed using Resume().
+ // Note that this method does not affect partition subscription.
+ // In particular, it does not cause a group rebalance when automatic assignment is used.
+ Pause()
+
+ // Resume resumes this partition which have been paused with Pause().
+ // New calls to the broker will return records from these partitions if there are any to be fetched.
+ // If the partition was not previously paused, this method is a no-op.
+ Resume()
+
+ // IsPaused indicates if this partition consumer is paused or not
+ IsPaused() bool
+}
+
+type partitionConsumer struct {
+ highWaterMarkOffset atomic.Int64 // must be at the top of the struct because https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+
+ consumer *consumer
+ conf *Config
+ broker *brokerConsumer
+ messages chan *ConsumerMessage
+ errors chan *ConsumerError
+ feeder chan *FetchResponse
+
+ leaderEpoch int32
+ preferredReadReplica int32
+
+ trigger, dying chan none
+ closeOnce sync.Once
+ topic string
+ partition int32
+ responseResult error
+ fetchSize int32
+ offset int64
+ retries atomic.Int32
+
+ paused atomic.Bool // accessed atomically, 0 = not paused, 1 = paused
+}
+
+var errTimedOut = errors.New("timed out feeding messages to the user") // not user-facing
+
+func (child *partitionConsumer) sendError(err error) {
+ cErr := &ConsumerError{
+ Topic: child.topic,
+ Partition: child.partition,
+ Err: err,
+ }
+
+ if child.conf.Consumer.Return.Errors {
+ child.errors <- cErr
+ } else {
+ Logger.Println(cErr)
+ }
+}
+
+func (child *partitionConsumer) computeBackoff() time.Duration {
+ if child.conf.Consumer.Retry.BackoffFunc != nil {
+ retries := child.retries.Add(1)
+ return child.conf.Consumer.Retry.BackoffFunc(int(retries))
+ }
+ return child.conf.Consumer.Retry.Backoff
+}
+
+func (child *partitionConsumer) dispatcher() {
+ for range child.trigger {
+ select {
+ case <-child.dying:
+ close(child.trigger)
+ case <-time.After(child.computeBackoff()):
+ if child.broker != nil {
+ child.consumer.unrefBrokerConsumer(child.broker)
+ child.broker = nil
+ }
+
+ if err := child.dispatch(); err != nil {
+ child.sendError(err)
+ child.trigger <- none{}
+ }
+ }
+ }
+
+ if child.broker != nil {
+ child.consumer.unrefBrokerConsumer(child.broker)
+ }
+ child.consumer.removeChild(child)
+ close(child.feeder)
+}
+
+func (child *partitionConsumer) preferredBroker() (*Broker, int32, error) {
+ if child.preferredReadReplica >= 0 {
+ broker, err := child.consumer.client.Broker(child.preferredReadReplica)
+ if err == nil {
+ return broker, child.leaderEpoch, nil
+ }
+ Logger.Printf(
+ "consumer/%s/%d failed to find active broker for preferred read replica %d - will fallback to leader",
+ child.topic, child.partition, child.preferredReadReplica)
+
+ // if we couldn't find it, discard the replica preference and trigger a
+ // metadata refresh whilst falling back to consuming from the leader again
+ child.preferredReadReplica = invalidPreferredReplicaID
+ _ = child.consumer.client.RefreshMetadata(child.topic)
+ }
+
+ // if preferred replica cannot be found fallback to leader
+ return child.consumer.client.LeaderAndEpoch(child.topic, child.partition)
+}
+
+func (child *partitionConsumer) dispatch() error {
+ if err := child.consumer.client.RefreshMetadata(child.topic); err != nil {
+ return err
+ }
+
+ broker, epoch, err := child.preferredBroker()
+ if err != nil {
+ return err
+ }
+
+ child.leaderEpoch = epoch
+ child.broker = child.consumer.refBrokerConsumer(broker)
+ child.broker.input <- child
+
+ return nil
+}
+
+func (child *partitionConsumer) chooseStartingOffset(offset int64) error {
+ newestOffset, err := child.consumer.client.GetOffset(child.topic, child.partition, OffsetNewest)
+ if err != nil {
+ return err
+ }
+
+ child.highWaterMarkOffset.Store(newestOffset)
+
+ oldestOffset, err := child.consumer.client.GetOffset(child.topic, child.partition, OffsetOldest)
+ if err != nil {
+ return err
+ }
+
+ switch {
+ case offset == OffsetNewest:
+ child.offset = newestOffset
+ case offset == OffsetOldest:
+ child.offset = oldestOffset
+ case offset >= oldestOffset && offset <= newestOffset:
+ child.offset = offset
+ default:
+ return ErrOffsetOutOfRange
+ }
+
+ return nil
+}
+
+func (child *partitionConsumer) Messages() <-chan *ConsumerMessage {
+ return child.messages
+}
+
+func (child *partitionConsumer) Errors() <-chan *ConsumerError {
+ return child.errors
+}
+
+func (child *partitionConsumer) AsyncClose() {
+ // this triggers whatever broker owns this child to abandon it and close its trigger channel, which causes
+ // the dispatcher to exit its loop, which removes it from the consumer then closes its 'messages' and
+ // 'errors' channel (alternatively, if the child is already at the dispatcher for some reason, that will
+ // also just close itself)
+ child.closeOnce.Do(func() {
+ close(child.dying)
+ })
+}
+
+func (child *partitionConsumer) Close() error {
+ child.AsyncClose()
+
+ var consumerErrors ConsumerErrors
+ for err := range child.errors {
+ consumerErrors = append(consumerErrors, err)
+ }
+
+ if len(consumerErrors) > 0 {
+ return consumerErrors
+ }
+ return nil
+}
+
+func (child *partitionConsumer) HighWaterMarkOffset() int64 {
+ return child.highWaterMarkOffset.Load()
+}
+
+func (child *partitionConsumer) responseFeeder() {
+ var msgs []*ConsumerMessage
+ expiryTicker := time.NewTicker(child.conf.Consumer.MaxProcessingTime)
+ firstAttempt := true
+
+feederLoop:
+ for response := range child.feeder {
+ msgs, child.responseResult = child.parseResponse(response)
+
+ if child.responseResult == nil {
+ child.retries.Store(0)
+ }
+
+ for i, msg := range msgs {
+ child.interceptors(msg)
+ messageSelect:
+ select {
+ case <-child.dying:
+ child.broker.acks.Done()
+ continue feederLoop
+ case child.messages <- msg:
+ firstAttempt = true
+ case <-expiryTicker.C:
+ if !firstAttempt {
+ child.responseResult = errTimedOut
+ child.broker.acks.Done()
+ remainingLoop:
+ for _, msg = range msgs[i:] {
+ child.interceptors(msg)
+ select {
+ case child.messages <- msg:
+ case <-child.dying:
+ break remainingLoop
+ }
+ }
+ child.broker.input <- child
+ continue feederLoop
+ } else {
+ // current message has not been sent, return to select
+ // statement
+ firstAttempt = false
+ goto messageSelect
+ }
+ }
+ }
+
+ child.broker.acks.Done()
+ }
+
+ expiryTicker.Stop()
+ close(child.messages)
+ close(child.errors)
+}
+
+func (child *partitionConsumer) parseMessages(msgSet *MessageSet) ([]*ConsumerMessage, error) {
+ var messages []*ConsumerMessage
+ for _, msgBlock := range msgSet.Messages {
+ for _, msg := range msgBlock.Messages() {
+ offset := msg.Offset
+ timestamp := msg.Msg.Timestamp
+ if msg.Msg.Version >= 1 {
+ baseOffset := msgBlock.Offset - msgBlock.Messages()[len(msgBlock.Messages())-1].Offset
+ offset += baseOffset
+ if msg.Msg.LogAppendTime {
+ timestamp = msgBlock.Msg.Timestamp
+ }
+ }
+ if offset < child.offset {
+ continue
+ }
+ messages = append(messages, &ConsumerMessage{
+ Topic: child.topic,
+ Partition: child.partition,
+ Key: msg.Msg.Key,
+ Value: msg.Msg.Value,
+ Offset: offset,
+ Timestamp: timestamp,
+ BlockTimestamp: msgBlock.Msg.Timestamp,
+ })
+ child.offset = offset + 1
+ }
+ }
+ if len(messages) == 0 {
+ child.offset++
+ }
+ return messages, nil
+}
+
+func (child *partitionConsumer) parseRecords(batch *RecordBatch) ([]*ConsumerMessage, error) {
+ messages := make([]*ConsumerMessage, 0, len(batch.Records))
+
+ for _, rec := range batch.Records {
+ offset := batch.FirstOffset + rec.OffsetDelta
+ if offset < child.offset {
+ continue
+ }
+ timestamp := batch.FirstTimestamp.Add(rec.TimestampDelta)
+ if batch.LogAppendTime {
+ timestamp = batch.MaxTimestamp
+ }
+ messages = append(messages, &ConsumerMessage{
+ Topic: child.topic,
+ Partition: child.partition,
+ Key: rec.Key,
+ Value: rec.Value,
+ Offset: offset,
+ Timestamp: timestamp,
+ Headers: rec.Headers,
+ })
+ child.offset = offset + 1
+ }
+ if len(messages) == 0 {
+ child.offset++
+ }
+ return messages, nil
+}
+
+func (child *partitionConsumer) parseResponse(response *FetchResponse) ([]*ConsumerMessage, error) {
+ var consumerBatchSizeMetric metrics.Histogram
+ if child.consumer != nil && child.consumer.metricRegistry != nil {
+ consumerBatchSizeMetric = getOrRegisterHistogram("consumer-batch-size", child.consumer.metricRegistry)
+ }
+
+ // If request was throttled and empty we log and return without error
+ if response.ThrottleTime != time.Duration(0) && len(response.Blocks) == 0 {
+ Logger.Printf(
+ "consumer/broker/%d FetchResponse throttled %v\n",
+ child.broker.broker.ID(), response.ThrottleTime)
+ return nil, nil
+ }
+
+ block := response.GetBlock(child.topic, child.partition)
+ if block == nil {
+ return nil, ErrIncompleteResponse
+ }
+
+ if !errors.Is(block.Err, ErrNoError) {
+ return nil, block.Err
+ }
+
+ nRecs, err := block.numRecords()
+ if err != nil {
+ return nil, err
+ }
+
+ if consumerBatchSizeMetric != nil {
+ consumerBatchSizeMetric.Update(int64(nRecs))
+ }
+
+ if block.PreferredReadReplica != invalidPreferredReplicaID {
+ child.preferredReadReplica = block.PreferredReadReplica
+ }
+
+ if nRecs == 0 {
+ partialTrailingMessage, err := block.isPartial()
+ if err != nil {
+ return nil, err
+ }
+ // We got no messages. If we got a trailing one then we need to ask for more data.
+ // Otherwise we just poll again and wait for one to be produced...
+ if partialTrailingMessage {
+ if child.conf.Consumer.Fetch.Max > 0 && child.fetchSize == child.conf.Consumer.Fetch.Max {
+ // we can't ask for more data, we've hit the configured limit
+ child.sendError(ErrMessageTooLarge)
+ child.offset++ // skip this one so we can keep processing future messages
+ } else {
+ child.fetchSize *= 2
+ // check int32 overflow
+ if child.fetchSize < 0 {
+ child.fetchSize = math.MaxInt32
+ }
+ if child.conf.Consumer.Fetch.Max > 0 && child.fetchSize > child.conf.Consumer.Fetch.Max {
+ child.fetchSize = child.conf.Consumer.Fetch.Max
+ }
+ }
+ } else if block.recordsNextOffset != nil && *block.recordsNextOffset <= block.HighWaterMarkOffset {
+ // check last record next offset to avoid stuck if high watermark was not reached
+ Logger.Printf("consumer/broker/%d received batch with zero records but high watermark was not reached, topic %s, partition %d, next offset %d\n", child.broker.broker.ID(), child.topic, child.partition, *block.recordsNextOffset)
+ child.offset = *block.recordsNextOffset
+ }
+
+ return nil, nil
+ }
+
+ // we got messages, reset our fetch size in case it was increased for a previous request
+ child.fetchSize = child.conf.Consumer.Fetch.Default
+ child.highWaterMarkOffset.Store(block.HighWaterMarkOffset)
+
+ // abortedProducerIDs contains producerID which message should be ignored as uncommitted
+ // - producerID are added when the partitionConsumer iterate over the offset at which an aborted transaction begins (abortedTransaction.FirstOffset)
+ // - producerID are removed when partitionConsumer iterate over an aborted controlRecord, meaning the aborted transaction for this producer is over
+ abortedProducerIDs := make(map[int64]struct{}, len(block.AbortedTransactions))
+ abortedTransactions := block.getAbortedTransactions()
+
+ var messages []*ConsumerMessage
+ for _, records := range block.RecordsSet {
+ switch records.recordsType {
+ case legacyRecords:
+ messageSetMessages, err := child.parseMessages(records.MsgSet)
+ if err != nil {
+ return nil, err
+ }
+
+ messages = append(messages, messageSetMessages...)
+ case defaultRecords:
+ // Consume remaining abortedTransaction up to last offset of current batch
+ for _, txn := range abortedTransactions {
+ if txn.FirstOffset > records.RecordBatch.LastOffset() {
+ break
+ }
+ abortedProducerIDs[txn.ProducerID] = struct{}{}
+ // Pop abortedTransactions so that we never add it again
+ abortedTransactions = abortedTransactions[1:]
+ }
+
+ recordBatchMessages, err := child.parseRecords(records.RecordBatch)
+ if err != nil {
+ return nil, err
+ }
+
+ // Parse and commit offset but do not expose messages that are:
+ // - control records
+ // - part of an aborted transaction when set to `ReadCommitted`
+
+ // control record
+ isControl, err := records.isControl()
+ if err != nil {
+ // I don't know why there is this continue in case of error to begin with
+ // Safe bet is to ignore control messages if ReadUncommitted
+ // and block on them in case of error and ReadCommitted
+ if child.conf.Consumer.IsolationLevel == ReadCommitted {
+ return nil, err
+ }
+ continue
+ }
+ if isControl {
+ controlRecord, err := records.getControlRecord()
+ if err != nil {
+ return nil, err
+ }
+
+ if controlRecord.Type == ControlRecordAbort {
+ delete(abortedProducerIDs, records.RecordBatch.ProducerID)
+ }
+ continue
+ }
+
+ // filter aborted transactions
+ if child.conf.Consumer.IsolationLevel == ReadCommitted {
+ _, isAborted := abortedProducerIDs[records.RecordBatch.ProducerID]
+ if records.RecordBatch.IsTransactional && isAborted {
+ continue
+ }
+ }
+
+ messages = append(messages, recordBatchMessages...)
+ default:
+ return nil, fmt.Errorf("unknown records type: %v", records.recordsType)
+ }
+ }
+
+ return messages, nil
+}
+
+func (child *partitionConsumer) interceptors(msg *ConsumerMessage) {
+ for _, interceptor := range child.conf.Consumer.Interceptors {
+ msg.safelyApplyInterceptor(interceptor)
+ }
+}
+
+// Pause implements PartitionConsumer.
+func (child *partitionConsumer) Pause() {
+ child.paused.Store(true)
+}
+
+// Resume implements PartitionConsumer.
+func (child *partitionConsumer) Resume() {
+ child.paused.Store(false)
+}
+
+// IsPaused implements PartitionConsumer.
+func (child *partitionConsumer) IsPaused() bool {
+ return child.paused.Load()
+}
+
+type brokerConsumer struct {
+ consumer *consumer
+ broker *Broker
+ input chan *partitionConsumer
+ newSubscriptions chan []*partitionConsumer
+ subscriptions map[*partitionConsumer]none
+ acks sync.WaitGroup
+ refs int
+}
+
+func (c *consumer) newBrokerConsumer(broker *Broker) *brokerConsumer {
+ bc := &brokerConsumer{
+ consumer: c,
+ broker: broker,
+ input: make(chan *partitionConsumer),
+ newSubscriptions: make(chan []*partitionConsumer),
+ subscriptions: make(map[*partitionConsumer]none),
+ refs: 0,
+ }
+
+ go withRecover(bc.subscriptionManager)
+ go withRecover(bc.subscriptionConsumer)
+
+ return bc
+}
+
+// The subscriptionManager constantly accepts new subscriptions on `input` (even when the main subscriptionConsumer
+// goroutine is in the middle of a network request) and batches it up. The main worker goroutine picks
+// up a batch of new subscriptions between every network request by reading from `newSubscriptions`, so we give
+// it nil if no new subscriptions are available.
+func (bc *brokerConsumer) subscriptionManager() {
+ defer close(bc.newSubscriptions)
+
+ for {
+ var partitionConsumers []*partitionConsumer
+
+ // Check for any partition consumer asking to subscribe if there aren't
+ // any, trigger the network request (to fetch Kafka messages) by sending "nil" to the
+ // newSubscriptions channel
+ select {
+ case pc, ok := <-bc.input:
+ if !ok {
+ return
+ }
+ partitionConsumers = append(partitionConsumers, pc)
+ case bc.newSubscriptions <- nil:
+ continue
+ }
+
+ // drain input of any further incoming subscriptions
+ timer := time.NewTimer(partitionConsumersBatchTimeout)
+ for batchComplete := false; !batchComplete; {
+ select {
+ case pc := <-bc.input:
+ partitionConsumers = append(partitionConsumers, pc)
+ case <-timer.C:
+ batchComplete = true
+ }
+ }
+ timer.Stop()
+
+ Logger.Printf(
+ "consumer/broker/%d accumulated %d new subscriptions\n",
+ bc.broker.ID(), len(partitionConsumers))
+
+ bc.newSubscriptions <- partitionConsumers
+ }
+}
+
+// subscriptionConsumer ensures we will get nil right away if no new subscriptions is available
+// this is the main loop that fetches Kafka messages
+func (bc *brokerConsumer) subscriptionConsumer() {
+ for newSubscriptions := range bc.newSubscriptions {
+ bc.updateSubscriptions(newSubscriptions)
+
+ if len(bc.subscriptions) == 0 {
+ // We're about to be shut down or we're about to receive more subscriptions.
+ // Take a small nap to avoid burning the CPU.
+ time.Sleep(partitionConsumersBatchTimeout)
+ continue
+ }
+
+ response, err := bc.fetchNewMessages()
+ if err != nil {
+ Logger.Printf("consumer/broker/%d disconnecting due to error processing FetchRequest: %s\n", bc.broker.ID(), err)
+ bc.abort(err)
+ return
+ }
+
+ // if there isn't response, it means that not fetch was made
+ // so we don't need to handle any response
+ if response == nil {
+ time.Sleep(partitionConsumersBatchTimeout)
+ continue
+ }
+
+ bc.acks.Add(len(bc.subscriptions))
+ for child := range bc.subscriptions {
+ if _, ok := response.Blocks[child.topic]; !ok {
+ bc.acks.Done()
+ continue
+ }
+
+ if _, ok := response.Blocks[child.topic][child.partition]; !ok {
+ bc.acks.Done()
+ continue
+ }
+
+ child.feeder <- response
+ }
+ bc.acks.Wait()
+ bc.handleResponses()
+ }
+}
+
+func (bc *brokerConsumer) updateSubscriptions(newSubscriptions []*partitionConsumer) {
+ for _, child := range newSubscriptions {
+ bc.subscriptions[child] = none{}
+ Logger.Printf("consumer/broker/%d added subscription to %s/%d\n", bc.broker.ID(), child.topic, child.partition)
+ }
+
+ for child := range bc.subscriptions {
+ select {
+ case <-child.dying:
+ Logger.Printf("consumer/broker/%d closed dead subscription to %s/%d\n", bc.broker.ID(), child.topic, child.partition)
+ close(child.trigger)
+ delete(bc.subscriptions, child)
+ default:
+ // no-op
+ }
+ }
+}
+
+// handleResponses handles the response codes left for us by our subscriptions, and abandons ones that have been closed
+func (bc *brokerConsumer) handleResponses() {
+ for child := range bc.subscriptions {
+ result := child.responseResult
+ child.responseResult = nil
+
+ if result == nil {
+ if preferredBroker, _, err := child.preferredBroker(); err == nil {
+ if bc.broker.ID() != preferredBroker.ID() {
+ // not an error but needs redispatching to consume from preferred replica
+ Logger.Printf(
+ "consumer/broker/%d abandoned in favor of preferred replica broker/%d\n",
+ bc.broker.ID(), preferredBroker.ID())
+ child.trigger <- none{}
+ delete(bc.subscriptions, child)
+ }
+ }
+ continue
+ }
+
+ // Discard any replica preference.
+ child.preferredReadReplica = invalidPreferredReplicaID
+
+ if errors.Is(result, errTimedOut) {
+ Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because consuming was taking too long\n",
+ bc.broker.ID(), child.topic, child.partition)
+ delete(bc.subscriptions, child)
+ } else if errors.Is(result, ErrOffsetOutOfRange) {
+ // there's no point in retrying this it will just fail the same way again
+ // shut it down and force the user to choose what to do
+ child.sendError(result)
+ Logger.Printf("consumer/%s/%d shutting down because %s\n", child.topic, child.partition, result)
+ close(child.trigger)
+ delete(bc.subscriptions, child)
+ } else if errors.Is(result, ErrUnknownTopicOrPartition) ||
+ errors.Is(result, ErrNotLeaderForPartition) ||
+ errors.Is(result, ErrLeaderNotAvailable) ||
+ errors.Is(result, ErrReplicaNotAvailable) ||
+ errors.Is(result, ErrFencedLeaderEpoch) ||
+ errors.Is(result, ErrUnknownLeaderEpoch) {
+ // not an error, but does need redispatching
+ Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because %s\n",
+ bc.broker.ID(), child.topic, child.partition, result)
+ child.trigger <- none{}
+ delete(bc.subscriptions, child)
+ } else {
+ // dunno, tell the user and try redispatching
+ child.sendError(result)
+ Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because %s\n",
+ bc.broker.ID(), child.topic, child.partition, result)
+ child.trigger <- none{}
+ delete(bc.subscriptions, child)
+ }
+ }
+}
+
+func (bc *brokerConsumer) abort(err error) {
+ bc.consumer.abandonBrokerConsumer(bc)
+ _ = bc.broker.Close() // we don't care about the error this might return, we already have one
+
+ for child := range bc.subscriptions {
+ child.sendError(err)
+ child.trigger <- none{}
+ }
+
+ for newSubscriptions := range bc.newSubscriptions {
+ if len(newSubscriptions) == 0 {
+ // Take a small nap to avoid burning the CPU.
+ time.Sleep(partitionConsumersBatchTimeout)
+ continue
+ }
+ for _, child := range newSubscriptions {
+ child.sendError(err)
+ child.trigger <- none{}
+ }
+ }
+}
+
+// fetchNewMessages can be nil if no fetch is made, it can occur when
+// all partitions are paused
+func (bc *brokerConsumer) fetchNewMessages() (*FetchResponse, error) {
+ request := &FetchRequest{
+ MinBytes: bc.consumer.conf.Consumer.Fetch.Min,
+ MaxWaitTime: int32(bc.consumer.conf.Consumer.MaxWaitTime / time.Millisecond),
+ }
+ // Version 1 is the same as version 0.
+ if bc.consumer.conf.Version.IsAtLeast(V0_9_0_0) {
+ request.Version = 1
+ }
+ // Starting in Version 2, the requestor must be able to handle Kafka Log
+ // Message format version 1.
+ if bc.consumer.conf.Version.IsAtLeast(V0_10_0_0) {
+ request.Version = 2
+ }
+ // Version 3 adds MaxBytes. Starting in version 3, the partition ordering in
+ // the request is now relevant. Partitions will be processed in the order
+ // they appear in the request.
+ if bc.consumer.conf.Version.IsAtLeast(V0_10_1_0) {
+ request.Version = 3
+ request.MaxBytes = MaxResponseSize
+ }
+ // Version 4 adds IsolationLevel. Starting in version 4, the reqestor must be
+ // able to handle Kafka log message format version 2.
+ // Version 5 adds LogStartOffset to indicate the earliest available offset of
+ // partition data that can be consumed.
+ if bc.consumer.conf.Version.IsAtLeast(V0_11_0_0) {
+ request.Version = 5
+ request.Isolation = bc.consumer.conf.Consumer.IsolationLevel
+ }
+ // Version 6 is the same as version 5.
+ if bc.consumer.conf.Version.IsAtLeast(V1_0_0_0) {
+ request.Version = 6
+ }
+ // Version 7 adds incremental fetch request support.
+ if bc.consumer.conf.Version.IsAtLeast(V1_1_0_0) {
+ request.Version = 7
+ // We do not currently implement KIP-227 FetchSessions. Setting the id to 0
+ // and the epoch to -1 tells the broker not to generate as session ID we're going
+ // to just ignore anyway.
+ request.SessionID = 0
+ request.SessionEpoch = -1
+ }
+ // Version 8 is the same as version 7.
+ if bc.consumer.conf.Version.IsAtLeast(V2_0_0_0) {
+ request.Version = 8
+ }
+ // Version 9 adds CurrentLeaderEpoch, as described in KIP-320.
+ // Version 10 indicates that we can use the ZStd compression algorithm, as
+ // described in KIP-110.
+ if bc.consumer.conf.Version.IsAtLeast(V2_1_0_0) {
+ request.Version = 10
+ }
+ // Version 11 adds RackID for KIP-392 fetch from closest replica
+ if bc.consumer.conf.Version.IsAtLeast(V2_3_0_0) {
+ request.Version = 11
+ request.RackID = bc.consumer.conf.RackID
+ }
+
+ for child := range bc.subscriptions {
+ if !child.IsPaused() {
+ request.AddBlock(child.topic, child.partition, child.offset, child.fetchSize, child.leaderEpoch)
+ }
+ }
+
+ // avoid to fetch when there is no block
+ if len(request.blocks) == 0 {
+ return nil, nil
+ }
+
+ return bc.broker.Fetch(request)
+}
diff --git a/vendor/github.com/IBM/sarama/consumer_group.go b/vendor/github.com/IBM/sarama/consumer_group.go
new file mode 100644
index 0000000..f47eac7
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/consumer_group.go
@@ -0,0 +1,1184 @@
+package sarama
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "sort"
+ "sync"
+ "time"
+
+ "github.com/rcrowley/go-metrics"
+)
+
+// ErrClosedConsumerGroup is the error returned when a method is called on a consumer group that has been closed.
+var ErrClosedConsumerGroup = errors.New("kafka: tried to use a consumer group that was closed")
+
+// ConsumerGroup is responsible for dividing up processing of topics and partitions
+// over a collection of processes (the members of the consumer group).
+type ConsumerGroup interface {
+ // Consume joins a cluster of consumers for a given list of topics and
+ // starts a blocking ConsumerGroupSession through the ConsumerGroupHandler.
+ //
+ // The life-cycle of a session is represented by the following steps:
+ //
+ // 1. The consumers join the group (as explained in https://kafka.apache.org/documentation/#intro_consumers)
+ // and is assigned their "fair share" of partitions, aka 'claims'.
+ // 2. Before processing starts, the handler's Setup() hook is called to notify the user
+ // of the claims and allow any necessary preparation or alteration of state.
+ // 3. For each of the assigned claims the handler's ConsumeClaim() function is then called
+ // in a separate goroutine which requires it to be thread-safe. Any state must be carefully protected
+ // from concurrent reads/writes.
+ // 4. The session will persist until one of the ConsumeClaim() functions exits. This can be either when the
+ // parent context is canceled or when a server-side rebalance cycle is initiated.
+ // 5. Once all the ConsumeClaim() loops have exited, the handler's Cleanup() hook is called
+ // to allow the user to perform any final tasks before a rebalance.
+ // 6. Finally, marked offsets are committed one last time before claims are released.
+ //
+ // Please note, that once a rebalance is triggered, sessions must be completed within
+ // Config.Consumer.Group.Rebalance.Timeout. This means that ConsumeClaim() functions must exit
+ // as quickly as possible to allow time for Cleanup() and the final offset commit. If the timeout
+ // is exceeded, the consumer will be removed from the group by Kafka, which will cause offset
+ // commit failures.
+ // This method should be called inside an infinite loop, when a
+ // server-side rebalance happens, the consumer session will need to be
+ // recreated to get the new claims.
+ Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error
+
+ // Errors returns a read channel of errors that occurred during the consumer life-cycle.
+ // By default, errors are logged and not returned over this channel.
+ // If you want to implement any custom error handling, set your config's
+ // Consumer.Return.Errors setting to true, and read from this channel.
+ Errors() <-chan error
+
+ // Close stops the ConsumerGroup and detaches any running sessions. It is required to call
+ // this function before the object passes out of scope, as it will otherwise leak memory.
+ Close() error
+
+ // Pause suspends fetching from the requested partitions. Future calls to the broker will not return any
+ // records from these partitions until they have been resumed using Resume()/ResumeAll().
+ // Note that this method does not affect partition subscription.
+ // In particular, it does not cause a group rebalance when automatic assignment is used.
+ Pause(partitions map[string][]int32)
+
+ // Resume resumes specified partitions which have been paused with Pause()/PauseAll().
+ // New calls to the broker will return records from these partitions if there are any to be fetched.
+ Resume(partitions map[string][]int32)
+
+ // Pause suspends fetching from all partitions. Future calls to the broker will not return any
+ // records from these partitions until they have been resumed using Resume()/ResumeAll().
+ // Note that this method does not affect partition subscription.
+ // In particular, it does not cause a group rebalance when automatic assignment is used.
+ PauseAll()
+
+ // Resume resumes all partitions which have been paused with Pause()/PauseAll().
+ // New calls to the broker will return records from these partitions if there are any to be fetched.
+ ResumeAll()
+}
+
+type consumerGroup struct {
+ client Client
+
+ config *Config
+ consumer Consumer
+ groupID string
+ groupInstanceId *string
+ memberID string
+ errors chan error
+
+ lock sync.Mutex
+ errorsLock sync.RWMutex
+ closed chan none
+ closeOnce sync.Once
+
+ userData []byte
+
+ metricRegistry metrics.Registry
+}
+
+// NewConsumerGroup creates a new consumer group the given broker addresses and configuration.
+func NewConsumerGroup(addrs []string, groupID string, config *Config) (ConsumerGroup, error) {
+ client, err := NewClient(addrs, config)
+ if err != nil {
+ return nil, err
+ }
+
+ c, err := newConsumerGroup(groupID, client)
+ if err != nil {
+ _ = client.Close()
+ }
+ return c, err
+}
+
+// NewConsumerGroupFromClient creates a new consumer group using the given client. It is still
+// necessary to call Close() on the underlying client when shutting down this consumer.
+// PLEASE NOTE: consumer groups can only re-use but not share clients.
+func NewConsumerGroupFromClient(groupID string, client Client) (ConsumerGroup, error) {
+ if client == nil {
+ return nil, ConfigurationError("client must not be nil")
+ }
+ // For clients passed in by the client, ensure we don't
+ // call Close() on it.
+ cli := &nopCloserClient{client}
+ return newConsumerGroup(groupID, cli)
+}
+
+func newConsumerGroup(groupID string, client Client) (ConsumerGroup, error) {
+ config := client.Config()
+ if !config.Version.IsAtLeast(V0_10_2_0) {
+ return nil, ConfigurationError("consumer groups require Version to be >= V0_10_2_0")
+ }
+
+ consumer, err := newConsumer(client)
+ if err != nil {
+ return nil, err
+ }
+
+ cg := &consumerGroup{
+ client: client,
+ consumer: consumer,
+ config: config,
+ groupID: groupID,
+ errors: make(chan error, config.ChannelBufferSize),
+ closed: make(chan none),
+ userData: config.Consumer.Group.Member.UserData,
+ metricRegistry: newCleanupRegistry(config.MetricRegistry),
+ }
+ if config.Consumer.Group.InstanceId != "" && config.Version.IsAtLeast(V2_3_0_0) {
+ cg.groupInstanceId = &config.Consumer.Group.InstanceId
+ }
+ return cg, nil
+}
+
+// Errors implements ConsumerGroup.
+func (c *consumerGroup) Errors() <-chan error { return c.errors }
+
+// Close implements ConsumerGroup.
+func (c *consumerGroup) Close() (err error) {
+ c.closeOnce.Do(func() {
+ close(c.closed)
+
+ // leave group
+ if e := c.leave(); e != nil {
+ err = e
+ }
+
+ go func() {
+ c.errorsLock.Lock()
+ defer c.errorsLock.Unlock()
+ close(c.errors)
+ }()
+
+ // drain errors
+ for e := range c.errors {
+ err = e
+ }
+
+ if e := c.client.Close(); e != nil {
+ err = e
+ }
+
+ c.metricRegistry.UnregisterAll()
+ })
+ return
+}
+
+// Consume implements ConsumerGroup.
+func (c *consumerGroup) Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error {
+ // Ensure group is not closed
+ select {
+ case <-c.closed:
+ return ErrClosedConsumerGroup
+ default:
+ }
+
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ // Quick exit when no topics are provided
+ if len(topics) == 0 {
+ return fmt.Errorf("no topics provided")
+ }
+
+ // Refresh metadata for requested topics
+ if err := c.client.RefreshMetadata(topics...); err != nil {
+ return err
+ }
+
+ // Init session
+ sess, err := c.newSession(ctx, topics, handler, c.config.Consumer.Group.Rebalance.Retry.Max)
+ if errors.Is(err, ErrClosedClient) {
+ return ErrClosedConsumerGroup
+ } else if err != nil {
+ return err
+ }
+
+ // Wait for session exit signal or Close() call
+ select {
+ case <-c.closed:
+ case <-sess.ctx.Done():
+ }
+
+ // Gracefully release session claims
+ return sess.release(true)
+}
+
+// Pause implements ConsumerGroup.
+func (c *consumerGroup) Pause(partitions map[string][]int32) {
+ c.consumer.Pause(partitions)
+}
+
+// Resume implements ConsumerGroup.
+func (c *consumerGroup) Resume(partitions map[string][]int32) {
+ c.consumer.Resume(partitions)
+}
+
+// PauseAll implements ConsumerGroup.
+func (c *consumerGroup) PauseAll() {
+ c.consumer.PauseAll()
+}
+
+// ResumeAll implements ConsumerGroup.
+func (c *consumerGroup) ResumeAll() {
+ c.consumer.ResumeAll()
+}
+
+func (c *consumerGroup) retryNewSession(ctx context.Context, topics []string, handler ConsumerGroupHandler, retries int, refreshCoordinator bool) (*consumerGroupSession, error) {
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-c.closed:
+ return nil, ErrClosedConsumerGroup
+ case <-time.After(c.config.Consumer.Group.Rebalance.Retry.Backoff):
+ }
+
+ if refreshCoordinator {
+ err := c.client.RefreshCoordinator(c.groupID)
+ if err != nil {
+ if retries <= 0 {
+ return nil, err
+ }
+ return c.retryNewSession(ctx, topics, handler, retries-1, true)
+ }
+ }
+
+ return c.newSession(ctx, topics, handler, retries-1)
+}
+
+func (c *consumerGroup) newSession(ctx context.Context, topics []string, handler ConsumerGroupHandler, retries int) (*consumerGroupSession, error) {
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ coordinator, err := c.client.Coordinator(c.groupID)
+ if err != nil {
+ if retries <= 0 {
+ return nil, err
+ }
+
+ return c.retryNewSession(ctx, topics, handler, retries, true)
+ }
+
+ var (
+ metricRegistry = c.metricRegistry
+ consumerGroupJoinTotal metrics.Counter
+ consumerGroupJoinFailed metrics.Counter
+ consumerGroupSyncTotal metrics.Counter
+ consumerGroupSyncFailed metrics.Counter
+ )
+
+ if metricRegistry != nil {
+ consumerGroupJoinTotal = metrics.GetOrRegisterCounter(fmt.Sprintf("consumer-group-join-total-%s", c.groupID), metricRegistry)
+ consumerGroupJoinFailed = metrics.GetOrRegisterCounter(fmt.Sprintf("consumer-group-join-failed-%s", c.groupID), metricRegistry)
+ consumerGroupSyncTotal = metrics.GetOrRegisterCounter(fmt.Sprintf("consumer-group-sync-total-%s", c.groupID), metricRegistry)
+ consumerGroupSyncFailed = metrics.GetOrRegisterCounter(fmt.Sprintf("consumer-group-sync-failed-%s", c.groupID), metricRegistry)
+ }
+
+ // Join consumer group
+ join, err := c.joinGroupRequest(coordinator, topics)
+ if consumerGroupJoinTotal != nil {
+ consumerGroupJoinTotal.Inc(1)
+ }
+ if err != nil {
+ _ = coordinator.Close()
+ if consumerGroupJoinFailed != nil {
+ consumerGroupJoinFailed.Inc(1)
+ }
+ return nil, err
+ }
+ if !errors.Is(join.Err, ErrNoError) {
+ if consumerGroupJoinFailed != nil {
+ consumerGroupJoinFailed.Inc(1)
+ }
+ }
+ switch join.Err {
+ case ErrNoError:
+ c.memberID = join.MemberId
+ case ErrUnknownMemberId, ErrIllegalGeneration:
+ // reset member ID and retry immediately
+ c.memberID = ""
+ return c.newSession(ctx, topics, handler, retries)
+ case ErrNotCoordinatorForConsumer, ErrRebalanceInProgress, ErrOffsetsLoadInProgress:
+ // retry after backoff
+ if retries <= 0 {
+ return nil, join.Err
+ }
+ return c.retryNewSession(ctx, topics, handler, retries, true)
+ case ErrMemberIdRequired:
+ // from JoinGroupRequest v4 onwards (due to KIP-394) if the client starts
+ // with an empty member id, it needs to get the assigned id from the
+ // response and send another join request with that id to actually join the
+ // group
+ c.memberID = join.MemberId
+ return c.newSession(ctx, topics, handler, retries)
+ case ErrFencedInstancedId:
+ if c.groupInstanceId != nil {
+ Logger.Printf("JoinGroup failed: group instance id %s has been fenced\n", *c.groupInstanceId)
+ }
+ return nil, join.Err
+ default:
+ return nil, join.Err
+ }
+
+ var strategy BalanceStrategy
+ var ok bool
+ if strategy = c.config.Consumer.Group.Rebalance.Strategy; strategy == nil {
+ strategy, ok = c.findStrategy(join.GroupProtocol, c.config.Consumer.Group.Rebalance.GroupStrategies)
+ if !ok {
+ // this case shouldn't happen in practice, since the leader will choose the protocol
+ // that all the members support
+ return nil, fmt.Errorf("unable to find selected strategy: %s", join.GroupProtocol)
+ }
+ }
+
+ // Prepare distribution plan if we joined as the leader
+ var plan BalanceStrategyPlan
+ var members map[string]ConsumerGroupMemberMetadata
+ var allSubscribedTopicPartitions map[string][]int32
+ var allSubscribedTopics []string
+ if join.LeaderId == join.MemberId {
+ members, err = join.GetMembers()
+ if err != nil {
+ return nil, err
+ }
+
+ allSubscribedTopicPartitions, allSubscribedTopics, plan, err = c.balance(strategy, members)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ // Sync consumer group
+ syncGroupResponse, err := c.syncGroupRequest(coordinator, members, plan, join.GenerationId, strategy)
+ if consumerGroupSyncTotal != nil {
+ consumerGroupSyncTotal.Inc(1)
+ }
+ if err != nil {
+ _ = coordinator.Close()
+ if consumerGroupSyncFailed != nil {
+ consumerGroupSyncFailed.Inc(1)
+ }
+ return nil, err
+ }
+ if !errors.Is(syncGroupResponse.Err, ErrNoError) {
+ if consumerGroupSyncFailed != nil {
+ consumerGroupSyncFailed.Inc(1)
+ }
+ }
+
+ switch syncGroupResponse.Err {
+ case ErrNoError:
+ case ErrUnknownMemberId, ErrIllegalGeneration:
+ // reset member ID and retry immediately
+ c.memberID = ""
+ return c.newSession(ctx, topics, handler, retries)
+ case ErrNotCoordinatorForConsumer, ErrRebalanceInProgress, ErrOffsetsLoadInProgress:
+ // retry after backoff
+ if retries <= 0 {
+ return nil, syncGroupResponse.Err
+ }
+ return c.retryNewSession(ctx, topics, handler, retries, true)
+ case ErrFencedInstancedId:
+ if c.groupInstanceId != nil {
+ Logger.Printf("JoinGroup failed: group instance id %s has been fenced\n", *c.groupInstanceId)
+ }
+ return nil, syncGroupResponse.Err
+ default:
+ return nil, syncGroupResponse.Err
+ }
+
+ // Retrieve and sort claims
+ var claims map[string][]int32
+ if len(syncGroupResponse.MemberAssignment) > 0 {
+ members, err := syncGroupResponse.GetMemberAssignment()
+ if err != nil {
+ return nil, err
+ }
+ claims = members.Topics
+
+ // in the case of stateful balance strategies, hold on to the returned
+ // assignment metadata, otherwise, reset the statically defined consumer
+ // group metadata
+ if members.UserData != nil {
+ c.userData = members.UserData
+ } else {
+ c.userData = c.config.Consumer.Group.Member.UserData
+ }
+
+ for _, partitions := range claims {
+ sort.Sort(int32Slice(partitions))
+ }
+ }
+
+ session, err := newConsumerGroupSession(ctx, c, claims, join.MemberId, join.GenerationId, handler)
+ if err != nil {
+ return nil, err
+ }
+
+ // only the leader needs to check whether there are newly-added partitions in order to trigger a rebalance
+ if join.LeaderId == join.MemberId {
+ go c.loopCheckPartitionNumbers(allSubscribedTopicPartitions, allSubscribedTopics, session)
+ }
+
+ return session, err
+}
+
+func (c *consumerGroup) joinGroupRequest(coordinator *Broker, topics []string) (*JoinGroupResponse, error) {
+ req := &JoinGroupRequest{
+ GroupId: c.groupID,
+ MemberId: c.memberID,
+ SessionTimeout: int32(c.config.Consumer.Group.Session.Timeout / time.Millisecond),
+ ProtocolType: "consumer",
+ }
+ if c.config.Version.IsAtLeast(V0_10_1_0) {
+ req.Version = 1
+ req.RebalanceTimeout = int32(c.config.Consumer.Group.Rebalance.Timeout / time.Millisecond)
+ }
+ if c.config.Version.IsAtLeast(V0_11_0_0) {
+ req.Version = 2
+ }
+ if c.config.Version.IsAtLeast(V0_11_0_0) {
+ req.Version = 2
+ }
+ if c.config.Version.IsAtLeast(V2_0_0_0) {
+ req.Version = 3
+ }
+ // from JoinGroupRequest v4 onwards (due to KIP-394) the client will actually
+ // send two JoinGroupRequests, once with the empty member id, and then again
+ // with the assigned id from the first response. This is handled via the
+ // ErrMemberIdRequired case.
+ if c.config.Version.IsAtLeast(V2_2_0_0) {
+ req.Version = 4
+ }
+ if c.config.Version.IsAtLeast(V2_3_0_0) {
+ req.Version = 5
+ req.GroupInstanceId = c.groupInstanceId
+ if c.config.Version.IsAtLeast(V2_4_0_0) {
+ req.Version = 6
+ }
+ }
+
+ meta := &ConsumerGroupMemberMetadata{
+ Topics: topics,
+ UserData: c.userData,
+ }
+ var strategy BalanceStrategy
+ if strategy = c.config.Consumer.Group.Rebalance.Strategy; strategy != nil {
+ if err := req.AddGroupProtocolMetadata(strategy.Name(), meta); err != nil {
+ return nil, err
+ }
+ } else {
+ for _, strategy = range c.config.Consumer.Group.Rebalance.GroupStrategies {
+ if err := req.AddGroupProtocolMetadata(strategy.Name(), meta); err != nil {
+ return nil, err
+ }
+ }
+ }
+
+ return coordinator.JoinGroup(req)
+}
+
+// findStrategy returns the BalanceStrategy with the specified protocolName
+// from the slice provided.
+func (c *consumerGroup) findStrategy(name string, groupStrategies []BalanceStrategy) (BalanceStrategy, bool) {
+ for _, strategy := range groupStrategies {
+ if strategy.Name() == name {
+ return strategy, true
+ }
+ }
+ return nil, false
+}
+
+func (c *consumerGroup) syncGroupRequest(
+ coordinator *Broker,
+ members map[string]ConsumerGroupMemberMetadata,
+ plan BalanceStrategyPlan,
+ generationID int32,
+ strategy BalanceStrategy,
+) (*SyncGroupResponse, error) {
+ req := &SyncGroupRequest{
+ GroupId: c.groupID,
+ MemberId: c.memberID,
+ GenerationId: generationID,
+ }
+
+ // Versions 1 and 2 are the same as version 0.
+ if c.config.Version.IsAtLeast(V0_11_0_0) {
+ req.Version = 1
+ }
+ if c.config.Version.IsAtLeast(V2_0_0_0) {
+ req.Version = 2
+ }
+ // Starting from version 3, we add a new field called groupInstanceId to indicate member identity across restarts.
+ if c.config.Version.IsAtLeast(V2_3_0_0) {
+ req.Version = 3
+ req.GroupInstanceId = c.groupInstanceId
+ if c.config.Version.IsAtLeast(V2_4_0_0) {
+ req.Version = 4
+ }
+ }
+
+ for memberID, topics := range plan {
+ assignment := &ConsumerGroupMemberAssignment{Topics: topics}
+ userDataBytes, err := strategy.AssignmentData(memberID, topics, generationID)
+ if err != nil {
+ return nil, err
+ }
+ assignment.UserData = userDataBytes
+ if err := req.AddGroupAssignmentMember(memberID, assignment); err != nil {
+ return nil, err
+ }
+ delete(members, memberID)
+ }
+ // add empty assignments for any remaining members
+ for memberID := range members {
+ if err := req.AddGroupAssignmentMember(memberID, &ConsumerGroupMemberAssignment{}); err != nil {
+ return nil, err
+ }
+ }
+
+ return coordinator.SyncGroup(req)
+}
+
+func (c *consumerGroup) heartbeatRequest(coordinator *Broker, memberID string, generationID int32) (*HeartbeatResponse, error) {
+ req := &HeartbeatRequest{
+ GroupId: c.groupID,
+ MemberId: memberID,
+ GenerationId: generationID,
+ }
+
+ // Version 1 and version 2 are the same as version 0.
+ if c.config.Version.IsAtLeast(V0_11_0_0) {
+ req.Version = 1
+ }
+ if c.config.Version.IsAtLeast(V2_0_0_0) {
+ req.Version = 2
+ }
+ // Starting from version 3, we add a new field called groupInstanceId to indicate member identity across restarts.
+ if c.config.Version.IsAtLeast(V2_3_0_0) {
+ req.Version = 3
+ req.GroupInstanceId = c.groupInstanceId
+ // Version 4 is the first flexible version
+ if c.config.Version.IsAtLeast(V2_4_0_0) {
+ req.Version = 4
+ }
+ }
+
+ return coordinator.Heartbeat(req)
+}
+
+func (c *consumerGroup) balance(strategy BalanceStrategy, members map[string]ConsumerGroupMemberMetadata) (map[string][]int32, []string, BalanceStrategyPlan, error) {
+ topicPartitions := make(map[string][]int32)
+ for _, meta := range members {
+ for _, topic := range meta.Topics {
+ topicPartitions[topic] = nil
+ }
+ }
+
+ allSubscribedTopics := make([]string, 0, len(topicPartitions))
+ for topic := range topicPartitions {
+ allSubscribedTopics = append(allSubscribedTopics, topic)
+ }
+
+ // refresh metadata for all the subscribed topics in the consumer group
+ // to avoid using stale metadata to assigning partitions
+ err := c.client.RefreshMetadata(allSubscribedTopics...)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+
+ for topic := range topicPartitions {
+ partitions, err := c.client.Partitions(topic)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ topicPartitions[topic] = partitions
+ }
+
+ plan, err := strategy.Plan(members, topicPartitions)
+ return topicPartitions, allSubscribedTopics, plan, err
+}
+
+// Leaves the cluster, called by Close.
+func (c *consumerGroup) leave() error {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+ if c.memberID == "" {
+ return nil
+ }
+
+ coordinator, err := c.client.Coordinator(c.groupID)
+ if err != nil {
+ return err
+ }
+
+ // as per KIP-345 if groupInstanceId is set, i.e. static membership is in action, then do not leave group when consumer closed, just clear memberID
+ if c.groupInstanceId != nil {
+ c.memberID = ""
+ return nil
+ }
+ req := &LeaveGroupRequest{
+ GroupId: c.groupID,
+ MemberId: c.memberID,
+ }
+ if c.config.Version.IsAtLeast(V0_11_0_0) {
+ req.Version = 1
+ }
+ if c.config.Version.IsAtLeast(V2_0_0_0) {
+ req.Version = 2
+ }
+ if c.config.Version.IsAtLeast(V2_4_0_0) {
+ req.Version = 4
+ req.Members = append(req.Members, MemberIdentity{
+ MemberId: c.memberID,
+ })
+ }
+
+ resp, err := coordinator.LeaveGroup(req)
+ if err != nil {
+ _ = coordinator.Close()
+ return err
+ }
+
+ // clear the memberID
+ c.memberID = ""
+
+ switch resp.Err {
+ case ErrRebalanceInProgress, ErrUnknownMemberId, ErrNoError:
+ return nil
+ default:
+ return resp.Err
+ }
+}
+
+func (c *consumerGroup) handleError(err error, topic string, partition int32) {
+ var consumerError *ConsumerError
+ if ok := errors.As(err, &consumerError); !ok && topic != "" && partition > -1 {
+ err = &ConsumerError{
+ Topic: topic,
+ Partition: partition,
+ Err: err,
+ }
+ }
+
+ if !c.config.Consumer.Return.Errors {
+ Logger.Println(err)
+ return
+ }
+
+ c.errorsLock.RLock()
+ defer c.errorsLock.RUnlock()
+ select {
+ case <-c.closed:
+ // consumer is closed
+ return
+ default:
+ }
+
+ select {
+ case c.errors <- err:
+ default:
+ // no error listener
+ }
+}
+
+func (c *consumerGroup) loopCheckPartitionNumbers(allSubscribedTopicPartitions map[string][]int32, topics []string, session *consumerGroupSession) {
+ if c.config.Metadata.RefreshFrequency == time.Duration(0) {
+ return
+ }
+
+ defer session.cancel()
+
+ oldTopicToPartitionNum := make(map[string]int, len(allSubscribedTopicPartitions))
+ for topic, partitions := range allSubscribedTopicPartitions {
+ oldTopicToPartitionNum[topic] = len(partitions)
+ }
+
+ pause := time.NewTicker(c.config.Metadata.RefreshFrequency)
+ defer pause.Stop()
+ for {
+ if newTopicToPartitionNum, err := c.topicToPartitionNumbers(topics); err != nil {
+ return
+ } else {
+ for topic, num := range oldTopicToPartitionNum {
+ if newTopicToPartitionNum[topic] != num {
+ Logger.Printf(
+ "consumergroup/%s loop check partition number goroutine find partitions in topics %s changed from %d to %d\n",
+ c.groupID, topics, num, newTopicToPartitionNum[topic])
+ return // trigger the end of the session on exit
+ }
+ }
+ }
+ select {
+ case <-pause.C:
+ case <-session.ctx.Done():
+ Logger.Printf(
+ "consumergroup/%s loop check partition number goroutine will exit, topics %s\n",
+ c.groupID, topics)
+ // if session closed by other, should be exited
+ return
+ case <-c.closed:
+ return
+ }
+ }
+}
+
+func (c *consumerGroup) topicToPartitionNumbers(topics []string) (map[string]int, error) {
+ topicToPartitionNum := make(map[string]int, len(topics))
+ for _, topic := range topics {
+ if partitionNum, err := c.client.Partitions(topic); err != nil {
+ Logger.Printf(
+ "consumergroup/%s topic %s get partition number failed due to '%v'\n",
+ c.groupID, topic, err)
+ return nil, err
+ } else {
+ topicToPartitionNum[topic] = len(partitionNum)
+ }
+ }
+ return topicToPartitionNum, nil
+}
+
+// --------------------------------------------------------------------
+
+// ConsumerGroupSession represents a consumer group member session.
+type ConsumerGroupSession interface {
+ // Claims returns information about the claimed partitions by topic.
+ Claims() map[string][]int32
+
+ // MemberID returns the cluster member ID.
+ MemberID() string
+
+ // GenerationID returns the current generation ID.
+ GenerationID() int32
+
+ // MarkOffset marks the provided offset, alongside a metadata string
+ // that represents the state of the partition consumer at that point in time. The
+ // metadata string can be used by another consumer to restore that state, so it
+ // can resume consumption.
+ //
+ // To follow upstream conventions, you are expected to mark the offset of the
+ // next message to read, not the last message read. Thus, when calling `MarkOffset`
+ // you should typically add one to the offset of the last consumed message.
+ //
+ // Note: calling MarkOffset does not necessarily commit the offset to the backend
+ // store immediately for efficiency reasons, and it may never be committed if
+ // your application crashes. This means that you may end up processing the same
+ // message twice, and your processing should ideally be idempotent.
+ MarkOffset(topic string, partition int32, offset int64, metadata string)
+
+ // Commit the offset to the backend
+ //
+ // Note: calling Commit performs a blocking synchronous operation.
+ Commit()
+
+ // ResetOffset resets to the provided offset, alongside a metadata string that
+ // represents the state of the partition consumer at that point in time. Reset
+ // acts as a counterpart to MarkOffset, the difference being that it allows to
+ // reset an offset to an earlier or smaller value, where MarkOffset only
+ // allows incrementing the offset. cf MarkOffset for more details.
+ ResetOffset(topic string, partition int32, offset int64, metadata string)
+
+ // MarkMessage marks a message as consumed.
+ MarkMessage(msg *ConsumerMessage, metadata string)
+
+ // Context returns the session context.
+ Context() context.Context
+}
+
+type consumerGroupSession struct {
+ parent *consumerGroup
+ memberID string
+ generationID int32
+ handler ConsumerGroupHandler
+
+ claims map[string][]int32
+ offsets *offsetManager
+ ctx context.Context
+ cancel func()
+
+ waitGroup sync.WaitGroup
+ releaseOnce sync.Once
+ hbDying, hbDead chan none
+}
+
+func newConsumerGroupSession(ctx context.Context, parent *consumerGroup, claims map[string][]int32, memberID string, generationID int32, handler ConsumerGroupHandler) (*consumerGroupSession, error) {
+ // init context
+ ctx, cancel := context.WithCancel(ctx)
+
+ // init offset manager
+ offsets, err := newOffsetManagerFromClient(parent.groupID, memberID, generationID, parent.client, cancel)
+ if err != nil {
+ return nil, err
+ }
+
+ // init session
+ sess := &consumerGroupSession{
+ parent: parent,
+ memberID: memberID,
+ generationID: generationID,
+ handler: handler,
+ offsets: offsets,
+ claims: claims,
+ ctx: ctx,
+ cancel: cancel,
+ hbDying: make(chan none),
+ hbDead: make(chan none),
+ }
+
+ // start heartbeat loop
+ go sess.heartbeatLoop()
+
+ // create a POM for each claim
+ for topic, partitions := range claims {
+ for _, partition := range partitions {
+ pom, err := offsets.ManagePartition(topic, partition)
+ if err != nil {
+ _ = sess.release(false)
+ return nil, err
+ }
+
+ // handle POM errors
+ go func(topic string, partition int32) {
+ for err := range pom.Errors() {
+ sess.parent.handleError(err, topic, partition)
+ }
+ }(topic, partition)
+ }
+ }
+
+ // perform setup
+ if err := handler.Setup(sess); err != nil {
+ _ = sess.release(true)
+ return nil, err
+ }
+
+ // start consuming each topic partition in its own goroutine
+ for topic, partitions := range claims {
+ for _, partition := range partitions {
+ sess.waitGroup.Add(1) // increment wait group before spawning goroutine
+ go func(topic string, partition int32) {
+ defer sess.waitGroup.Done()
+ // cancel the group session as soon as any of the consume calls return
+ defer sess.cancel()
+
+ // if partition not currently readable, wait for it to become readable
+ if sess.parent.client.PartitionNotReadable(topic, partition) {
+ timer := time.NewTimer(5 * time.Second)
+ defer timer.Stop()
+
+ for sess.parent.client.PartitionNotReadable(topic, partition) {
+ select {
+ case <-ctx.Done():
+ return
+ case <-parent.closed:
+ return
+ case <-timer.C:
+ timer.Reset(5 * time.Second)
+ }
+ }
+ }
+
+ // consume a single topic/partition, blocking
+ sess.consume(topic, partition)
+ }(topic, partition)
+ }
+ }
+ return sess, nil
+}
+
+func (s *consumerGroupSession) Claims() map[string][]int32 { return s.claims }
+func (s *consumerGroupSession) MemberID() string { return s.memberID }
+func (s *consumerGroupSession) GenerationID() int32 { return s.generationID }
+
+func (s *consumerGroupSession) MarkOffset(topic string, partition int32, offset int64, metadata string) {
+ if pom := s.offsets.findPOM(topic, partition); pom != nil {
+ pom.MarkOffset(offset, metadata)
+ }
+}
+
+func (s *consumerGroupSession) Commit() {
+ s.offsets.Commit()
+}
+
+func (s *consumerGroupSession) ResetOffset(topic string, partition int32, offset int64, metadata string) {
+ if pom := s.offsets.findPOM(topic, partition); pom != nil {
+ pom.ResetOffset(offset, metadata)
+ }
+}
+
+func (s *consumerGroupSession) MarkMessage(msg *ConsumerMessage, metadata string) {
+ s.MarkOffset(msg.Topic, msg.Partition, msg.Offset+1, metadata)
+}
+
+func (s *consumerGroupSession) Context() context.Context {
+ return s.ctx
+}
+
+func (s *consumerGroupSession) consume(topic string, partition int32) {
+ // quick exit if rebalance is due
+ select {
+ case <-s.ctx.Done():
+ return
+ case <-s.parent.closed:
+ return
+ default:
+ }
+
+ // get next offset
+ offset := s.parent.config.Consumer.Offsets.Initial
+ if pom := s.offsets.findPOM(topic, partition); pom != nil {
+ offset, _ = pom.NextOffset()
+ }
+
+ // create new claim
+ claim, err := newConsumerGroupClaim(s, topic, partition, offset)
+ if err != nil {
+ s.parent.handleError(err, topic, partition)
+ return
+ }
+
+ // handle errors
+ go func() {
+ for err := range claim.Errors() {
+ s.parent.handleError(err, topic, partition)
+ }
+ }()
+
+ // trigger close when session is done
+ go func() {
+ select {
+ case <-s.ctx.Done():
+ case <-s.parent.closed:
+ }
+ claim.AsyncClose()
+ }()
+
+ // start processing
+ if err := s.handler.ConsumeClaim(s, claim); err != nil {
+ s.parent.handleError(err, topic, partition)
+ }
+
+ // ensure consumer is closed & drained
+ claim.AsyncClose()
+ for _, err := range claim.waitClosed() {
+ s.parent.handleError(err, topic, partition)
+ }
+}
+
+func (s *consumerGroupSession) release(withCleanup bool) (err error) {
+ // signal release, stop heartbeat
+ s.cancel()
+
+ // wait for consumers to exit
+ s.waitGroup.Wait()
+
+ // perform release
+ s.releaseOnce.Do(func() {
+ if withCleanup {
+ if e := s.handler.Cleanup(s); e != nil {
+ s.parent.handleError(e, "", -1)
+ err = e
+ }
+ }
+
+ if e := s.offsets.Close(); e != nil {
+ err = e
+ }
+
+ close(s.hbDying)
+ <-s.hbDead
+ })
+
+ Logger.Printf(
+ "consumergroup/session/%s/%d released\n",
+ s.MemberID(), s.GenerationID())
+
+ return
+}
+
+func (s *consumerGroupSession) heartbeatLoop() {
+ defer close(s.hbDead)
+ defer s.cancel() // trigger the end of the session on exit
+ defer func() {
+ Logger.Printf(
+ "consumergroup/session/%s/%d heartbeat loop stopped\n",
+ s.MemberID(), s.GenerationID())
+ }()
+
+ pause := time.NewTicker(s.parent.config.Consumer.Group.Heartbeat.Interval)
+ defer pause.Stop()
+
+ retryBackoff := time.NewTimer(s.parent.config.Metadata.Retry.Backoff)
+ defer retryBackoff.Stop()
+
+ retries := s.parent.config.Metadata.Retry.Max
+ for {
+ coordinator, err := s.parent.client.Coordinator(s.parent.groupID)
+ if err != nil {
+ if retries <= 0 {
+ s.parent.handleError(err, "", -1)
+ return
+ }
+ retryBackoff.Reset(s.parent.config.Metadata.Retry.Backoff)
+ select {
+ case <-s.hbDying:
+ return
+ case <-retryBackoff.C:
+ retries--
+ }
+ continue
+ }
+
+ resp, err := s.parent.heartbeatRequest(coordinator, s.memberID, s.generationID)
+ if err != nil {
+ _ = coordinator.Close()
+
+ if retries <= 0 {
+ s.parent.handleError(err, "", -1)
+ return
+ }
+
+ retries--
+ continue
+ }
+
+ switch resp.Err {
+ case ErrNoError:
+ retries = s.parent.config.Metadata.Retry.Max
+ case ErrRebalanceInProgress:
+ retries = s.parent.config.Metadata.Retry.Max
+ s.cancel()
+ case ErrUnknownMemberId, ErrIllegalGeneration:
+ return
+ case ErrFencedInstancedId:
+ if s.parent.groupInstanceId != nil {
+ Logger.Printf("JoinGroup failed: group instance id %s has been fenced\n", *s.parent.groupInstanceId)
+ }
+ s.parent.handleError(resp.Err, "", -1)
+ return
+ default:
+ s.parent.handleError(resp.Err, "", -1)
+ return
+ }
+
+ select {
+ case <-pause.C:
+ case <-s.hbDying:
+ return
+ }
+ }
+}
+
+// --------------------------------------------------------------------
+
+// ConsumerGroupHandler instances are used to handle individual topic/partition claims.
+// It also provides hooks for your consumer group session life-cycle and allow you to
+// trigger logic before or after the consume loop(s).
+//
+// PLEASE NOTE that handlers are likely be called from several goroutines concurrently,
+// ensure that all state is safely protected against race conditions.
+type ConsumerGroupHandler interface {
+ // Setup is run at the beginning of a new session, before ConsumeClaim.
+ Setup(ConsumerGroupSession) error
+
+ // Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited
+ // but before the offsets are committed for the very last time.
+ Cleanup(ConsumerGroupSession) error
+
+ // ConsumeClaim must start a consumer loop of ConsumerGroupClaim's Messages().
+ // Once the Messages() channel is closed, the Handler must finish its processing
+ // loop and exit.
+ ConsumeClaim(ConsumerGroupSession, ConsumerGroupClaim) error
+}
+
+// ConsumerGroupClaim processes Kafka messages from a given topic and partition within a consumer group.
+type ConsumerGroupClaim interface {
+ // Topic returns the consumed topic name.
+ Topic() string
+
+ // Partition returns the consumed partition.
+ Partition() int32
+
+ // InitialOffset returns the initial offset that was used as a starting point for this claim.
+ InitialOffset() int64
+
+ // HighWaterMarkOffset returns the high watermark offset of the partition,
+ // i.e. the offset that will be used for the next message that will be produced.
+ // You can use this to determine how far behind the processing is.
+ HighWaterMarkOffset() int64
+
+ // Messages returns the read channel for the messages that are returned by
+ // the broker. The messages channel will be closed when a new rebalance cycle
+ // is due. You must finish processing and mark offsets within
+ // Config.Consumer.Group.Session.Timeout before the topic/partition is eventually
+ // re-assigned to another group member.
+ Messages() <-chan *ConsumerMessage
+}
+
+type consumerGroupClaim struct {
+ topic string
+ partition int32
+ offset int64
+ PartitionConsumer
+}
+
+func newConsumerGroupClaim(sess *consumerGroupSession, topic string, partition int32, offset int64) (*consumerGroupClaim, error) {
+ pcm, err := sess.parent.consumer.ConsumePartition(topic, partition, offset)
+
+ if errors.Is(err, ErrOffsetOutOfRange) && sess.parent.config.Consumer.Group.ResetInvalidOffsets {
+ offset = sess.parent.config.Consumer.Offsets.Initial
+ pcm, err = sess.parent.consumer.ConsumePartition(topic, partition, offset)
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ go func() {
+ for err := range pcm.Errors() {
+ sess.parent.handleError(err, topic, partition)
+ }
+ }()
+
+ return &consumerGroupClaim{
+ topic: topic,
+ partition: partition,
+ offset: offset,
+ PartitionConsumer: pcm,
+ }, nil
+}
+
+func (c *consumerGroupClaim) Topic() string { return c.topic }
+func (c *consumerGroupClaim) Partition() int32 { return c.partition }
+func (c *consumerGroupClaim) InitialOffset() int64 { return c.offset }
+
+// Drains messages and errors, ensures the claim is fully closed.
+func (c *consumerGroupClaim) waitClosed() (errs ConsumerErrors) {
+ go func() {
+ for range c.Messages() {
+ }
+ }()
+
+ for err := range c.Errors() {
+ errs = append(errs, err)
+ }
+ return
+}
diff --git a/vendor/github.com/IBM/sarama/consumer_group_members.go b/vendor/github.com/IBM/sarama/consumer_group_members.go
new file mode 100644
index 0000000..2d38960
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/consumer_group_members.go
@@ -0,0 +1,183 @@
+package sarama
+
+import "errors"
+
+// ConsumerGroupMemberMetadata holds the metadata for consumer group
+// https://github.com/apache/kafka/blob/trunk/clients/src/main/resources/common/message/ConsumerProtocolSubscription.json
+type ConsumerGroupMemberMetadata struct {
+ Version int16
+ Topics []string
+ UserData []byte
+ OwnedPartitions []*OwnedPartition
+ GenerationID int32
+ RackID *string
+}
+
+func (m *ConsumerGroupMemberMetadata) encode(pe packetEncoder) error {
+ pe.putInt16(m.Version)
+
+ if err := pe.putStringArray(m.Topics); err != nil {
+ return err
+ }
+
+ if err := pe.putBytes(m.UserData); err != nil {
+ return err
+ }
+
+ if m.Version >= 1 {
+ if err := pe.putArrayLength(len(m.OwnedPartitions)); err != nil {
+ return err
+ }
+ for _, op := range m.OwnedPartitions {
+ if err := op.encode(pe); err != nil {
+ return err
+ }
+ }
+ }
+
+ if m.Version >= 2 {
+ pe.putInt32(m.GenerationID)
+ }
+
+ if m.Version >= 3 {
+ if err := pe.putNullableString(m.RackID); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (m *ConsumerGroupMemberMetadata) decode(pd packetDecoder) (err error) {
+ if m.Version, err = pd.getInt16(); err != nil {
+ return
+ }
+
+ if m.Topics, err = pd.getStringArray(); err != nil {
+ return
+ }
+
+ if m.UserData, err = pd.getBytes(); err != nil {
+ return
+ }
+ if m.Version >= 1 {
+ n, err := pd.getArrayLength()
+ if err != nil {
+ // permit missing data here in case of misbehaving 3rd party
+ // clients who incorrectly marked the member metadata as V1 in
+ // their JoinGroup request
+ if errors.Is(err, ErrInsufficientData) {
+ return nil
+ }
+ return err
+ }
+ if n > 0 {
+ m.OwnedPartitions = make([]*OwnedPartition, n)
+ for i := 0; i < n; i++ {
+ m.OwnedPartitions[i] = &OwnedPartition{}
+ if err := m.OwnedPartitions[i].decode(pd); err != nil {
+ return err
+ }
+ }
+ }
+ }
+
+ if m.Version >= 2 {
+ if m.GenerationID, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+
+ if m.Version >= 3 {
+ if m.RackID, err = pd.getNullableString(); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+type OwnedPartition struct {
+ Topic string
+ Partitions []int32
+}
+
+func (m *OwnedPartition) encode(pe packetEncoder) error {
+ if err := pe.putString(m.Topic); err != nil {
+ return err
+ }
+ if err := pe.putInt32Array(m.Partitions); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (m *OwnedPartition) decode(pd packetDecoder) (err error) {
+ if m.Topic, err = pd.getString(); err != nil {
+ return err
+ }
+ if m.Partitions, err = pd.getInt32Array(); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// ConsumerGroupMemberAssignment holds the member assignment for a consume group
+// https://github.com/apache/kafka/blob/trunk/clients/src/main/resources/common/message/ConsumerProtocolAssignment.json
+type ConsumerGroupMemberAssignment struct {
+ Version int16
+ Topics map[string][]int32
+ UserData []byte
+}
+
+func (m *ConsumerGroupMemberAssignment) encode(pe packetEncoder) error {
+ pe.putInt16(m.Version)
+
+ if err := pe.putArrayLength(len(m.Topics)); err != nil {
+ return err
+ }
+
+ for topic, partitions := range m.Topics {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putInt32Array(partitions); err != nil {
+ return err
+ }
+ }
+
+ if err := pe.putBytes(m.UserData); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (m *ConsumerGroupMemberAssignment) decode(pd packetDecoder) (err error) {
+ if m.Version, err = pd.getInt16(); err != nil {
+ return
+ }
+
+ var topicLen int
+ if topicLen, err = pd.getArrayLength(); err != nil {
+ return
+ }
+
+ m.Topics = make(map[string][]int32, topicLen)
+ for i := 0; i < topicLen; i++ {
+ var topic string
+ if topic, err = pd.getString(); err != nil {
+ return
+ }
+ if m.Topics[topic], err = pd.getInt32Array(); err != nil {
+ return
+ }
+ }
+
+ if m.UserData, err = pd.getBytes(); err != nil {
+ return
+ }
+
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/consumer_metadata_request.go b/vendor/github.com/IBM/sarama/consumer_metadata_request.go
new file mode 100644
index 0000000..85ffb03
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/consumer_metadata_request.go
@@ -0,0 +1,55 @@
+package sarama
+
+// ConsumerMetadataRequest is used for metadata requests
+type ConsumerMetadataRequest struct {
+ Version int16
+ ConsumerGroup string
+}
+
+func (r *ConsumerMetadataRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *ConsumerMetadataRequest) encode(pe packetEncoder) error {
+ tmp := new(FindCoordinatorRequest)
+ tmp.CoordinatorKey = r.ConsumerGroup
+ tmp.CoordinatorType = CoordinatorGroup
+ tmp.Version = r.Version
+ return tmp.encode(pe)
+}
+
+func (r *ConsumerMetadataRequest) decode(pd packetDecoder, version int16) (err error) {
+ tmp := new(FindCoordinatorRequest)
+ if err := tmp.decode(pd, version); err != nil {
+ return err
+ }
+ r.ConsumerGroup = tmp.CoordinatorKey
+ return nil
+}
+
+func (r *ConsumerMetadataRequest) key() int16 {
+ return apiKeyFindCoordinator
+}
+
+func (r *ConsumerMetadataRequest) version() int16 {
+ return r.Version
+}
+
+func (r *ConsumerMetadataRequest) headerVersion() int16 {
+ return 1
+}
+
+func (r *ConsumerMetadataRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 2
+}
+
+func (r *ConsumerMetadataRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ default:
+ return V0_8_2_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/consumer_metadata_response.go b/vendor/github.com/IBM/sarama/consumer_metadata_response.go
new file mode 100644
index 0000000..7fb080b
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/consumer_metadata_response.go
@@ -0,0 +1,98 @@
+package sarama
+
+import (
+ "net"
+ "strconv"
+)
+
+// ConsumerMetadataResponse holds the response for a consumer group meta data requests
+type ConsumerMetadataResponse struct {
+ Version int16
+ Err KError
+ Coordinator *Broker
+ CoordinatorID int32 // deprecated: use Coordinator.ID()
+ CoordinatorHost string // deprecated: use Coordinator.Addr()
+ CoordinatorPort int32 // deprecated: use Coordinator.Addr()
+}
+
+func (r *ConsumerMetadataResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *ConsumerMetadataResponse) decode(pd packetDecoder, version int16) (err error) {
+ tmp := new(FindCoordinatorResponse)
+
+ if err := tmp.decode(pd, version); err != nil {
+ return err
+ }
+
+ r.Err = tmp.Err
+
+ r.Coordinator = tmp.Coordinator
+ if tmp.Coordinator == nil {
+ return nil
+ }
+
+ // this can all go away in 2.0, but we have to fill in deprecated fields to maintain
+ // backwards compatibility
+ host, portstr, err := net.SplitHostPort(r.Coordinator.Addr())
+ if err != nil {
+ return err
+ }
+ port, err := strconv.ParseInt(portstr, 10, 32)
+ if err != nil {
+ return err
+ }
+ r.CoordinatorID = r.Coordinator.ID()
+ r.CoordinatorHost = host
+ r.CoordinatorPort = int32(port)
+
+ return nil
+}
+
+func (r *ConsumerMetadataResponse) encode(pe packetEncoder) error {
+ if r.Coordinator == nil {
+ r.Coordinator = new(Broker)
+ r.Coordinator.id = r.CoordinatorID
+ r.Coordinator.addr = net.JoinHostPort(r.CoordinatorHost, strconv.Itoa(int(r.CoordinatorPort)))
+ }
+
+ tmp := &FindCoordinatorResponse{
+ Version: r.Version,
+ Err: r.Err,
+ Coordinator: r.Coordinator,
+ }
+
+ if err := tmp.encode(pe); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (r *ConsumerMetadataResponse) key() int16 {
+ return apiKeyFindCoordinator
+}
+
+func (r *ConsumerMetadataResponse) version() int16 {
+ return r.Version
+}
+
+func (r *ConsumerMetadataResponse) headerVersion() int16 {
+ return 0
+}
+
+func (r *ConsumerMetadataResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 2
+}
+
+func (r *ConsumerMetadataResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ default:
+ return V0_8_2_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/control_record.go b/vendor/github.com/IBM/sarama/control_record.go
new file mode 100644
index 0000000..244a821
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/control_record.go
@@ -0,0 +1,74 @@
+package sarama
+
+// ControlRecordType ...
+type ControlRecordType int
+
+const (
+ // ControlRecordAbort is a control record for abort
+ ControlRecordAbort ControlRecordType = iota
+ // ControlRecordCommit is a control record for commit
+ ControlRecordCommit
+ // ControlRecordUnknown is a control record of unknown type
+ ControlRecordUnknown
+)
+
+// Control records are returned as a record by fetchRequest
+// However unlike "normal" records, they mean nothing application wise.
+// They only serve internal logic for supporting transactions.
+type ControlRecord struct {
+ Version int16
+ CoordinatorEpoch int32
+ Type ControlRecordType
+}
+
+func (cr *ControlRecord) decode(key, value packetDecoder) error {
+ var err error
+ // There a version for the value part AND the key part. And I have no idea if they are supposed to match or not
+ // Either way, all these version can only be 0 for now
+ cr.Version, err = key.getInt16()
+ if err != nil {
+ return err
+ }
+
+ recordType, err := key.getInt16()
+ if err != nil {
+ return err
+ }
+
+ switch recordType {
+ case 0:
+ cr.Type = ControlRecordAbort
+ case 1:
+ cr.Type = ControlRecordCommit
+ default:
+ // from JAVA implementation:
+ // UNKNOWN is used to indicate a control type which the client is not aware of and should be ignored
+ cr.Type = ControlRecordUnknown
+ }
+ // we want to parse value only if we are decoding control record of known type
+ if cr.Type != ControlRecordUnknown {
+ cr.Version, err = value.getInt16()
+ if err != nil {
+ return err
+ }
+
+ cr.CoordinatorEpoch, err = value.getInt32()
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (cr *ControlRecord) encode(key, value packetEncoder) {
+ value.putInt16(cr.Version)
+ value.putInt32(cr.CoordinatorEpoch)
+ key.putInt16(cr.Version)
+
+ switch cr.Type {
+ case ControlRecordAbort:
+ key.putInt16(0)
+ case ControlRecordCommit:
+ key.putInt16(1)
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/crc32_field.go b/vendor/github.com/IBM/sarama/crc32_field.go
new file mode 100644
index 0000000..32236e5
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/crc32_field.go
@@ -0,0 +1,87 @@
+package sarama
+
+import (
+ "encoding/binary"
+ "fmt"
+ "hash/crc32"
+ "sync"
+)
+
+type crcPolynomial int8
+
+const (
+ crcIEEE crcPolynomial = iota
+ crcCastagnoli
+)
+
+var crc32FieldPool = sync.Pool{}
+
+func acquireCrc32Field(polynomial crcPolynomial) *crc32Field {
+ val := crc32FieldPool.Get()
+ if val != nil {
+ c := val.(*crc32Field)
+ c.polynomial = polynomial
+ return c
+ }
+ return newCRC32Field(polynomial)
+}
+
+func releaseCrc32Field(c *crc32Field) {
+ crc32FieldPool.Put(c)
+}
+
+var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
+
+// crc32Field implements the pushEncoder and pushDecoder interfaces for calculating CRC32s.
+type crc32Field struct {
+ startOffset int
+ polynomial crcPolynomial
+}
+
+func (c *crc32Field) saveOffset(in int) {
+ c.startOffset = in
+}
+
+func (c *crc32Field) reserveLength() int {
+ return 4
+}
+
+func newCRC32Field(polynomial crcPolynomial) *crc32Field {
+ return &crc32Field{polynomial: polynomial}
+}
+
+func (c *crc32Field) run(curOffset int, buf []byte) error {
+ crc, err := c.crc(curOffset, buf)
+ if err != nil {
+ return err
+ }
+ binary.BigEndian.PutUint32(buf[c.startOffset:], crc)
+ return nil
+}
+
+func (c *crc32Field) check(curOffset int, buf []byte) error {
+ crc, err := c.crc(curOffset, buf)
+ if err != nil {
+ return err
+ }
+
+ expected := binary.BigEndian.Uint32(buf[c.startOffset:])
+ if crc != expected {
+ return PacketDecodingError{fmt.Sprintf("CRC didn't match expected %#x got %#x", expected, crc)}
+ }
+
+ return nil
+}
+
+func (c *crc32Field) crc(curOffset int, buf []byte) (uint32, error) {
+ var tab *crc32.Table
+ switch c.polynomial {
+ case crcIEEE:
+ tab = crc32.IEEETable
+ case crcCastagnoli:
+ tab = castagnoliTable
+ default:
+ return 0, PacketDecodingError{"invalid CRC type"}
+ }
+ return crc32.Checksum(buf[c.startOffset+4:curOffset], tab), nil
+}
diff --git a/vendor/github.com/IBM/sarama/create_partitions_request.go b/vendor/github.com/IBM/sarama/create_partitions_request.go
new file mode 100644
index 0000000..75e2dec
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/create_partitions_request.go
@@ -0,0 +1,141 @@
+package sarama
+
+import "time"
+
+type CreatePartitionsRequest struct {
+ Version int16
+ TopicPartitions map[string]*TopicPartition
+ Timeout time.Duration
+ ValidateOnly bool
+}
+
+func (c *CreatePartitionsRequest) setVersion(v int16) {
+ c.Version = v
+}
+
+func (c *CreatePartitionsRequest) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(c.TopicPartitions)); err != nil {
+ return err
+ }
+
+ for topic, partition := range c.TopicPartitions {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := partition.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ pe.putInt32(int32(c.Timeout / time.Millisecond))
+
+ pe.putBool(c.ValidateOnly)
+
+ return nil
+}
+
+func (c *CreatePartitionsRequest) decode(pd packetDecoder, version int16) (err error) {
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ c.TopicPartitions = make(map[string]*TopicPartition, n)
+ for i := 0; i < n; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ c.TopicPartitions[topic] = new(TopicPartition)
+ if err := c.TopicPartitions[topic].decode(pd, version); err != nil {
+ return err
+ }
+ }
+
+ timeout, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ c.Timeout = time.Duration(timeout) * time.Millisecond
+
+ if c.ValidateOnly, err = pd.getBool(); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (r *CreatePartitionsRequest) key() int16 {
+ return apiKeyCreatePartitions
+}
+
+func (r *CreatePartitionsRequest) version() int16 {
+ return r.Version
+}
+
+func (r *CreatePartitionsRequest) headerVersion() int16 {
+ return 1
+}
+
+func (r *CreatePartitionsRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 1
+}
+
+func (r *CreatePartitionsRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 1:
+ return V2_0_0_0
+ case 0:
+ return V1_0_0_0
+ default:
+ return V2_0_0_0
+ }
+}
+
+type TopicPartition struct {
+ Count int32
+ Assignment [][]int32
+}
+
+func (t *TopicPartition) encode(pe packetEncoder) error {
+ pe.putInt32(t.Count)
+
+ if len(t.Assignment) == 0 {
+ pe.putInt32(-1)
+ return nil
+ }
+
+ if err := pe.putArrayLength(len(t.Assignment)); err != nil {
+ return err
+ }
+
+ for _, assign := range t.Assignment {
+ if err := pe.putInt32Array(assign); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (t *TopicPartition) decode(pd packetDecoder, version int16) (err error) {
+ if t.Count, err = pd.getInt32(); err != nil {
+ return err
+ }
+
+ n, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ if n <= 0 {
+ return nil
+ }
+ t.Assignment = make([][]int32, n)
+
+ for i := 0; i < int(n); i++ {
+ if t.Assignment[i], err = pd.getInt32Array(); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/create_partitions_response.go b/vendor/github.com/IBM/sarama/create_partitions_response.go
new file mode 100644
index 0000000..0785c35
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/create_partitions_response.go
@@ -0,0 +1,130 @@
+package sarama
+
+import (
+ "fmt"
+ "time"
+)
+
+type CreatePartitionsResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ TopicPartitionErrors map[string]*TopicPartitionError
+}
+
+func (c *CreatePartitionsResponse) setVersion(v int16) {
+ c.Version = v
+}
+
+func (c *CreatePartitionsResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(c.ThrottleTime)
+ if err := pe.putArrayLength(len(c.TopicPartitionErrors)); err != nil {
+ return err
+ }
+
+ for topic, partitionError := range c.TopicPartitionErrors {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := partitionError.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (c *CreatePartitionsResponse) decode(pd packetDecoder, version int16) (err error) {
+ if c.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ c.TopicPartitionErrors = make(map[string]*TopicPartitionError, n)
+ for i := 0; i < n; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ c.TopicPartitionErrors[topic] = new(TopicPartitionError)
+ if err := c.TopicPartitionErrors[topic].decode(pd, version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (r *CreatePartitionsResponse) key() int16 {
+ return apiKeyCreatePartitions
+}
+
+func (r *CreatePartitionsResponse) version() int16 {
+ return r.Version
+}
+
+func (r *CreatePartitionsResponse) headerVersion() int16 {
+ return 0
+}
+
+func (r *CreatePartitionsResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 1
+}
+
+func (r *CreatePartitionsResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 1:
+ return V2_0_0_0
+ case 0:
+ return V1_0_0_0
+ default:
+ return V2_0_0_0
+ }
+}
+
+func (r *CreatePartitionsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
+
+type TopicPartitionError struct {
+ Err KError
+ ErrMsg *string
+}
+
+func (t *TopicPartitionError) Error() string {
+ text := t.Err.Error()
+ if t.ErrMsg != nil {
+ text = fmt.Sprintf("%s - %s", text, *t.ErrMsg)
+ }
+ return text
+}
+
+func (t *TopicPartitionError) Unwrap() error {
+ return t.Err
+}
+
+func (t *TopicPartitionError) encode(pe packetEncoder) error {
+ pe.putKError(t.Err)
+
+ if err := pe.putNullableString(t.ErrMsg); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (t *TopicPartitionError) decode(pd packetDecoder, version int16) (err error) {
+ t.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if t.ErrMsg, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/create_topics_request.go b/vendor/github.com/IBM/sarama/create_topics_request.go
new file mode 100644
index 0000000..5684e27
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/create_topics_request.go
@@ -0,0 +1,258 @@
+package sarama
+
+import (
+ "time"
+)
+
+type CreateTopicsRequest struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // TopicDetails contains the topics to create.
+ TopicDetails map[string]*TopicDetail
+ // Timeout contains how long to wait before timing out the request.
+ Timeout time.Duration
+ // ValidateOnly if true, check that the topics can be created as specified,
+ // but don't create anything.
+ ValidateOnly bool
+}
+
+func (c *CreateTopicsRequest) setVersion(v int16) {
+ c.Version = v
+}
+
+func NewCreateTopicsRequest(
+ version KafkaVersion,
+ topicDetails map[string]*TopicDetail,
+ timeout time.Duration,
+ validateOnly bool,
+) *CreateTopicsRequest {
+ r := &CreateTopicsRequest{
+ TopicDetails: topicDetails,
+ Timeout: timeout,
+ ValidateOnly: validateOnly,
+ }
+ switch {
+ case version.IsAtLeast(V2_4_0_0):
+ // Version 5 is the first flexible version
+ // Version 4 makes partitions/replicationFactor optional even when assignments are not present (KIP-464)
+ r.Version = 5
+ case version.IsAtLeast(V2_0_0_0):
+ // Version 3 is the same as version 2 (brokers response before throttling)
+ r.Version = 3
+ case version.IsAtLeast(V0_11_0_0):
+ // Version 2 is the same as version 1 (response has ThrottleTime)
+ r.Version = 2
+ case version.IsAtLeast(V0_10_2_0):
+ // Version 1 adds validateOnly.
+ r.Version = 1
+ }
+ return r
+}
+
+func (c *CreateTopicsRequest) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(c.TopicDetails)); err != nil {
+ return err
+ }
+ for topic, detail := range c.TopicDetails {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := detail.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ pe.putInt32(int32(c.Timeout / time.Millisecond))
+
+ if c.Version >= 1 {
+ pe.putBool(c.ValidateOnly)
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (c *CreateTopicsRequest) decode(pd packetDecoder, version int16) (err error) {
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ c.TopicDetails = make(map[string]*TopicDetail, n)
+
+ for i := 0; i < n; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ c.TopicDetails[topic] = new(TopicDetail)
+ if err = c.TopicDetails[topic].decode(pd, version); err != nil {
+ return err
+ }
+ }
+
+ timeout, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ c.Timeout = time.Duration(timeout) * time.Millisecond
+
+ if version >= 1 {
+ c.ValidateOnly, err = pd.getBool()
+ if err != nil {
+ return err
+ }
+
+ c.Version = version
+ }
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (c *CreateTopicsRequest) key() int16 {
+ return apiKeyCreateTopics
+}
+
+func (c *CreateTopicsRequest) version() int16 {
+ return c.Version
+}
+
+func (c *CreateTopicsRequest) headerVersion() int16 {
+ if c.Version >= 5 {
+ return 2
+ }
+ return 1
+}
+
+func (c *CreateTopicsRequest) isFlexible() bool {
+ return c.isFlexibleVersion(c.Version)
+}
+
+func (c *CreateTopicsRequest) isFlexibleVersion(version int16) bool {
+ return version >= 5
+}
+
+func (c *CreateTopicsRequest) isValidVersion() bool {
+ return c.Version >= 0 && c.Version <= 5
+}
+
+func (c *CreateTopicsRequest) requiredVersion() KafkaVersion {
+ switch c.Version {
+ case 5:
+ return V2_4_0_0
+ case 4:
+ return V2_4_0_0
+ case 3:
+ return V2_0_0_0
+ case 2:
+ return V0_11_0_0
+ case 1:
+ return V0_10_2_0
+ case 0:
+ return V0_10_1_0
+ default:
+ return V2_8_0_0
+ }
+}
+
+type TopicDetail struct {
+ // NumPartitions contains the number of partitions to create in the topic, or
+ // -1 if we are either specifying a manual partition assignment or using the
+ // default partitions.
+ NumPartitions int32
+ // ReplicationFactor contains the number of replicas to create for each
+ // partition in the topic, or -1 if we are either specifying a manual
+ // partition assignment or using the default replication factor.
+ ReplicationFactor int16
+ // ReplicaAssignment contains the manual partition assignment, or the empty
+ // array if we are using automatic assignment.
+ ReplicaAssignment map[int32][]int32
+ // ConfigEntries contains the custom topic configurations to set.
+ ConfigEntries map[string]*string
+}
+
+func (t *TopicDetail) encode(pe packetEncoder) error {
+ pe.putInt32(t.NumPartitions)
+ pe.putInt16(t.ReplicationFactor)
+
+ if err := pe.putArrayLength(len(t.ReplicaAssignment)); err != nil {
+ return err
+ }
+ for partition, assignment := range t.ReplicaAssignment {
+ pe.putInt32(partition)
+ if err := pe.putInt32Array(assignment); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ if err := pe.putArrayLength(len(t.ConfigEntries)); err != nil {
+ return err
+ }
+ for configKey, configValue := range t.ConfigEntries {
+ if err := pe.putString(configKey); err != nil {
+ return err
+ }
+ if err := pe.putNullableString(configValue); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (t *TopicDetail) decode(pd packetDecoder, version int16) (err error) {
+ if t.NumPartitions, err = pd.getInt32(); err != nil {
+ return err
+ }
+ if t.ReplicationFactor, err = pd.getInt16(); err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if n > 0 {
+ t.ReplicaAssignment = make(map[int32][]int32, n)
+ for i := 0; i < n; i++ {
+ replica, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ if t.ReplicaAssignment[replica], err = pd.getInt32Array(); err != nil {
+ return err
+ }
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+ }
+
+ n, err = pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if n > 0 {
+ t.ConfigEntries = make(map[string]*string, n)
+ for i := 0; i < n; i++ {
+ configKey, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ if t.ConfigEntries[configKey], err = pd.getNullableString(); err != nil {
+ return err
+ }
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
diff --git a/vendor/github.com/IBM/sarama/create_topics_response.go b/vendor/github.com/IBM/sarama/create_topics_response.go
new file mode 100644
index 0000000..f8be692
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/create_topics_response.go
@@ -0,0 +1,320 @@
+package sarama
+
+import (
+ "fmt"
+ "time"
+)
+
+type CreateTopicsResponse struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // ThrottleTime contains the duration for which the request was throttled due
+ // to a quota violation, or zero if the request did not violate any quota.
+ ThrottleTime time.Duration
+ // TopicErrors contains a map of any errors for the topics we tried to create.
+ TopicErrors map[string]*TopicError
+ // TopicResults contains a map of the results for the topics we tried to create.
+ TopicResults map[string]*CreatableTopicResult
+}
+
+func (c *CreateTopicsResponse) setVersion(v int16) {
+ c.Version = v
+}
+
+func (c *CreateTopicsResponse) encode(pe packetEncoder) error {
+ if c.Version >= 2 {
+ pe.putDurationMs(c.ThrottleTime)
+ }
+
+ if err := pe.putArrayLength(len(c.TopicErrors)); err != nil {
+ return err
+ }
+ for topic, topicError := range c.TopicErrors {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := topicError.encode(pe, c.Version); err != nil {
+ return err
+ }
+ if c.Version >= 5 {
+ result, ok := c.TopicResults[topic]
+ if !ok {
+ return fmt.Errorf("expected TopicResult for topic, %s, for V5 protocol", topic)
+ }
+ if err := result.encode(pe, c.Version); err != nil {
+ return err
+ }
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (c *CreateTopicsResponse) decode(pd packetDecoder, version int16) (err error) {
+ c.Version = version
+
+ if version >= 2 {
+ if c.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ c.TopicErrors = make(map[string]*TopicError, n)
+ if version >= 5 {
+ c.TopicResults = make(map[string]*CreatableTopicResult, n)
+ }
+ for i := 0; i < n; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ c.TopicErrors[topic] = new(TopicError)
+ if err := c.TopicErrors[topic].decode(pd, version); err != nil {
+ return err
+ }
+ if version >= 5 {
+ c.TopicResults[topic] = &CreatableTopicResult{}
+ if err := c.TopicResults[topic].decode(pd, version); err != nil {
+ return err
+ }
+ }
+ }
+
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (c *CreateTopicsResponse) key() int16 {
+ return apiKeyCreateTopics
+}
+
+func (c *CreateTopicsResponse) version() int16 {
+ return c.Version
+}
+
+func (c *CreateTopicsResponse) headerVersion() int16 {
+ if c.Version >= 5 {
+ return 1
+ }
+ return 0
+}
+
+func (c *CreateTopicsResponse) isFlexible() bool {
+ return c.isFlexibleVersion(c.Version)
+}
+
+func (c *CreateTopicsResponse) isFlexibleVersion(version int16) bool {
+ return version >= 5
+}
+
+func (c *CreateTopicsResponse) isValidVersion() bool {
+ return c.Version >= 0 && c.Version <= 5
+}
+
+func (c *CreateTopicsResponse) requiredVersion() KafkaVersion {
+ switch c.Version {
+ case 5:
+ return V2_4_0_0
+ case 4:
+ return V2_4_0_0
+ case 3:
+ return V2_0_0_0
+ case 2:
+ return V0_11_0_0
+ case 1:
+ return V0_10_2_0
+ case 0:
+ return V0_10_1_0
+ default:
+ return V2_8_0_0
+ }
+}
+
+func (r *CreateTopicsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
+
+type TopicError struct {
+ Err KError
+ ErrMsg *string
+}
+
+func (t *TopicError) Error() string {
+ text := t.Err.Error()
+ if t.ErrMsg != nil {
+ text = fmt.Sprintf("%s - %s", text, *t.ErrMsg)
+ }
+ return text
+}
+
+func (t *TopicError) Unwrap() error {
+ return t.Err
+}
+
+func (t *TopicError) encode(pe packetEncoder, version int16) error {
+ pe.putKError(t.Err)
+
+ if version >= 1 {
+ if err := pe.putNullableString(t.ErrMsg); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (t *TopicError) decode(pd packetDecoder, version int16) (err error) {
+ t.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if version >= 1 {
+ if t.ErrMsg, err = pd.getNullableString(); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// CreatableTopicResult struct {
+type CreatableTopicResult struct {
+ // TopicConfigErrorCode contains a Optional topic config error returned if configs are not returned in the response.
+ TopicConfigErrorCode KError
+ // NumPartitions contains a Number of partitions of the topic.
+ NumPartitions int32
+ // ReplicationFactor contains a Replication factor of the topic.
+ ReplicationFactor int16
+ // Configs contains a Configuration of the topic.
+ Configs map[string]*CreatableTopicConfigs
+}
+
+func (r *CreatableTopicResult) encode(pe packetEncoder, version int16) error {
+ pe.putInt32(r.NumPartitions)
+ pe.putInt16(r.ReplicationFactor)
+
+ if err := pe.putArrayLength(len(r.Configs)); err != nil {
+ return err
+ }
+ for name, config := range r.Configs {
+ if err := pe.putString(name); err != nil {
+ return err
+ }
+ if err := config.encode(pe, version); err != nil {
+ return err
+ }
+ }
+ if r.TopicConfigErrorCode == ErrNoError {
+ pe.putEmptyTaggedFieldArray()
+ return nil
+ }
+
+ // TODO: refactor to helper for tagged fields
+ pe.putUVarint(1) // number of tagged fields
+
+ pe.putUVarint(0) // tag
+
+ pe.putUVarint(2) // value length
+
+ pe.putKError(r.TopicConfigErrorCode) // tag value
+
+ return nil
+}
+
+func (r *CreatableTopicResult) decode(pd packetDecoder, version int16) (err error) {
+ r.NumPartitions, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ r.ReplicationFactor, err = pd.getInt16()
+ if err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ r.Configs = make(map[string]*CreatableTopicConfigs, n)
+ for i := 0; i < n; i++ {
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ r.Configs[name] = &CreatableTopicConfigs{}
+ if err := r.Configs[name].decode(pd, version); err != nil {
+ return err
+ }
+ }
+ err = pd.getTaggedFieldArray(taggedFieldDecoders{
+ 0: func(pd packetDecoder) error {
+ r.TopicConfigErrorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+ return nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+// CreatableTopicConfigs contains a Configuration of the topic.
+type CreatableTopicConfigs struct {
+ // Value contains the configuration value.
+ Value *string
+ // ReadOnly contains a True if the configuration is read-only.
+ ReadOnly bool
+ // ConfigSource contains the configuration source.
+ ConfigSource ConfigSource
+ // IsSensitive contains a True if this configuration is sensitive.
+ IsSensitive bool
+}
+
+func (c *CreatableTopicConfigs) encode(pe packetEncoder, version int16) (err error) {
+ if err = pe.putNullableString(c.Value); err != nil {
+ return err
+ }
+ pe.putBool(c.ReadOnly)
+ pe.putInt8(int8(c.ConfigSource))
+ pe.putBool(c.IsSensitive)
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (c *CreatableTopicConfigs) decode(pd packetDecoder, version int16) (err error) {
+ c.Value, err = pd.getNullableString()
+ if err != nil {
+ return err
+ }
+ c.ReadOnly, err = pd.getBool()
+ if err != nil {
+ return err
+ }
+ source, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ c.ConfigSource = ConfigSource(source)
+ c.IsSensitive, err = pd.getBool()
+ if err != nil {
+ return err
+ }
+
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/decompress.go b/vendor/github.com/IBM/sarama/decompress.go
new file mode 100644
index 0000000..0a09983
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/decompress.go
@@ -0,0 +1,98 @@
+package sarama
+
+import (
+ "bytes"
+ "fmt"
+ "sync"
+
+ snappy "github.com/eapache/go-xerial-snappy"
+ "github.com/klauspost/compress/gzip"
+ "github.com/pierrec/lz4/v4"
+)
+
+var (
+ lz4ReaderPool = sync.Pool{
+ New: func() interface{} {
+ return lz4.NewReader(nil)
+ },
+ }
+
+ gzipReaderPool sync.Pool
+
+ bufferPool = sync.Pool{
+ New: func() interface{} {
+ return new(bytes.Buffer)
+ },
+ }
+
+ bytesPool = sync.Pool{
+ New: func() interface{} {
+ res := make([]byte, 0, 4096)
+ return &res
+ },
+ }
+)
+
+func decompress(cc CompressionCodec, data []byte) ([]byte, error) {
+ switch cc {
+ case CompressionNone:
+ return data, nil
+ case CompressionGZIP:
+ var err error
+ reader, ok := gzipReaderPool.Get().(*gzip.Reader)
+ if !ok {
+ reader, err = gzip.NewReader(bytes.NewReader(data))
+ } else {
+ err = reader.Reset(bytes.NewReader(data))
+ }
+
+ if err != nil {
+ return nil, err
+ }
+
+ buffer := bufferPool.Get().(*bytes.Buffer)
+ _, err = buffer.ReadFrom(reader)
+ // copy the buffer to a new slice with the correct length
+ // reuse gzipReader and buffer
+ gzipReaderPool.Put(reader)
+ res := make([]byte, buffer.Len())
+ copy(res, buffer.Bytes())
+ buffer.Reset()
+ bufferPool.Put(buffer)
+
+ return res, err
+ case CompressionSnappy:
+ return snappy.Decode(data)
+ case CompressionLZ4:
+ reader, ok := lz4ReaderPool.Get().(*lz4.Reader)
+ if !ok {
+ reader = lz4.NewReader(bytes.NewReader(data))
+ } else {
+ reader.Reset(bytes.NewReader(data))
+ }
+ buffer := bufferPool.Get().(*bytes.Buffer)
+ _, err := buffer.ReadFrom(reader)
+ // copy the buffer to a new slice with the correct length
+ // reuse lz4Reader and buffer
+ lz4ReaderPool.Put(reader)
+ res := make([]byte, buffer.Len())
+ copy(res, buffer.Bytes())
+ buffer.Reset()
+ bufferPool.Put(buffer)
+
+ return res, err
+ case CompressionZSTD:
+ buffer := *bytesPool.Get().(*[]byte)
+ var err error
+ buffer, err = zstdDecompress(ZstdDecoderParams{}, buffer, data)
+ // copy the buffer to a new slice with the correct length and reuse buffer
+ res := make([]byte, len(buffer))
+ copy(res, buffer)
+ buffer = buffer[:0]
+ bytesPool.Put(&buffer)
+
+ return res, err
+ default:
+ return nil, PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", cc)}
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/delete_groups_request.go b/vendor/github.com/IBM/sarama/delete_groups_request.go
new file mode 100644
index 0000000..e6c1db0
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/delete_groups_request.go
@@ -0,0 +1,71 @@
+package sarama
+
+type DeleteGroupsRequest struct {
+ Version int16
+ Groups []string
+}
+
+func (r *DeleteGroupsRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *DeleteGroupsRequest) encode(pe packetEncoder) error {
+ if err := pe.putStringArray(r.Groups); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *DeleteGroupsRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Groups, err = pd.getStringArray()
+ if err != nil {
+ return err
+ }
+ _, err = pd.getEmptyTaggedFieldArray()
+ return
+}
+
+func (r *DeleteGroupsRequest) key() int16 {
+ return apiKeyDeleteGroups
+}
+
+func (r *DeleteGroupsRequest) version() int16 {
+ return r.Version
+}
+
+func (r *DeleteGroupsRequest) headerVersion() int16 {
+ if r.Version >= 2 {
+ return 2
+ }
+ return 1
+}
+
+func (r *DeleteGroupsRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *DeleteGroupsRequest) isFlexibleVersion(version int16) bool {
+ return version >= 2
+}
+
+func (r *DeleteGroupsRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 2
+}
+
+func (r *DeleteGroupsRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 2:
+ return V2_4_0_0
+ case 1:
+ return V2_0_0_0
+ case 0:
+ return V1_1_0_0
+ default:
+ return V2_0_0_0
+ }
+}
+
+func (r *DeleteGroupsRequest) AddGroup(group string) {
+ r.Groups = append(r.Groups, group)
+}
diff --git a/vendor/github.com/IBM/sarama/delete_groups_response.go b/vendor/github.com/IBM/sarama/delete_groups_response.go
new file mode 100644
index 0000000..ca683cd
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/delete_groups_response.go
@@ -0,0 +1,111 @@
+package sarama
+
+import (
+ "time"
+)
+
+type DeleteGroupsResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ GroupErrorCodes map[string]KError
+}
+
+func (r *DeleteGroupsResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *DeleteGroupsResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(r.ThrottleTime)
+
+ if err := pe.putArrayLength(len(r.GroupErrorCodes)); err != nil {
+ return err
+ }
+ for groupID, errorCode := range r.GroupErrorCodes {
+ if err := pe.putString(groupID); err != nil {
+ return err
+ }
+ pe.putKError(errorCode)
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *DeleteGroupsResponse) decode(pd packetDecoder, version int16) (err error) {
+ if r.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if n == 0 {
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+ }
+
+ r.GroupErrorCodes = make(map[string]KError, n)
+ for i := 0; i < n; i++ {
+ groupID, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ r.GroupErrorCodes[groupID], err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *DeleteGroupsResponse) key() int16 {
+ return apiKeyDeleteGroups
+}
+
+func (r *DeleteGroupsResponse) version() int16 {
+ return r.Version
+}
+
+func (r *DeleteGroupsResponse) headerVersion() int16 {
+ if r.Version >= 2 {
+ return 1
+ }
+ return 0
+}
+
+func (r *DeleteGroupsResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *DeleteGroupsResponse) isFlexibleVersion(version int16) bool {
+ return version >= 2
+}
+
+func (r *DeleteGroupsResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 2
+}
+
+func (r *DeleteGroupsResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 2:
+ return V2_4_0_0
+ case 1:
+ return V2_0_0_0
+ case 0:
+ return V1_1_0_0
+ default:
+ return V2_0_0_0
+ }
+}
+
+func (r *DeleteGroupsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/delete_offsets_request.go b/vendor/github.com/IBM/sarama/delete_offsets_request.go
new file mode 100644
index 0000000..8fd10ea
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/delete_offsets_request.go
@@ -0,0 +1,101 @@
+package sarama
+
+type DeleteOffsetsRequest struct {
+ Version int16
+ Group string
+ partitions map[string][]int32
+}
+
+func (r *DeleteOffsetsRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *DeleteOffsetsRequest) encode(pe packetEncoder) (err error) {
+ err = pe.putString(r.Group)
+ if err != nil {
+ return err
+ }
+
+ if r.partitions == nil {
+ pe.putInt32(0)
+ } else {
+ if err = pe.putArrayLength(len(r.partitions)); err != nil {
+ return err
+ }
+ }
+ for topic, partitions := range r.partitions {
+ err = pe.putString(topic)
+ if err != nil {
+ return err
+ }
+ err = pe.putInt32Array(partitions)
+ if err != nil {
+ return err
+ }
+ }
+ return
+}
+
+func (r *DeleteOffsetsRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Group, err = pd.getString()
+ if err != nil {
+ return err
+ }
+ var partitionCount int
+
+ partitionCount, err = pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if (partitionCount == 0 && version < 2) || partitionCount < 0 {
+ return nil
+ }
+
+ r.partitions = make(map[string][]int32, partitionCount)
+ for i := 0; i < partitionCount; i++ {
+ var topic string
+ topic, err = pd.getString()
+ if err != nil {
+ return err
+ }
+
+ var partitions []int32
+ partitions, err = pd.getInt32Array()
+ if err != nil {
+ return err
+ }
+
+ r.partitions[topic] = partitions
+ }
+
+ return nil
+}
+
+func (r *DeleteOffsetsRequest) key() int16 {
+ return apiKeyOffsetDelete
+}
+
+func (r *DeleteOffsetsRequest) version() int16 {
+ return r.Version
+}
+
+func (r *DeleteOffsetsRequest) headerVersion() int16 {
+ return 1
+}
+
+func (r *DeleteOffsetsRequest) isValidVersion() bool {
+ return r.Version == 0
+}
+
+func (r *DeleteOffsetsRequest) requiredVersion() KafkaVersion {
+ return V2_4_0_0
+}
+
+func (r *DeleteOffsetsRequest) AddPartition(topic string, partitionID int32) {
+ if r.partitions == nil {
+ r.partitions = make(map[string][]int32)
+ }
+
+ r.partitions[topic] = append(r.partitions[topic], partitionID)
+}
diff --git a/vendor/github.com/IBM/sarama/delete_offsets_response.go b/vendor/github.com/IBM/sarama/delete_offsets_response.go
new file mode 100644
index 0000000..64180b1
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/delete_offsets_response.go
@@ -0,0 +1,121 @@
+package sarama
+
+import (
+ "time"
+)
+
+type DeleteOffsetsResponse struct {
+ Version int16
+ // The top-level error code, or 0 if there was no error.
+ ErrorCode KError
+ ThrottleTime time.Duration
+ // The responses for each partition of the topics.
+ Errors map[string]map[int32]KError
+}
+
+func (r *DeleteOffsetsResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *DeleteOffsetsResponse) AddError(topic string, partition int32, errorCode KError) {
+ if r.Errors == nil {
+ r.Errors = make(map[string]map[int32]KError)
+ }
+ partitions := r.Errors[topic]
+ if partitions == nil {
+ partitions = make(map[int32]KError)
+ r.Errors[topic] = partitions
+ }
+ partitions[partition] = errorCode
+}
+
+func (r *DeleteOffsetsResponse) encode(pe packetEncoder) error {
+ pe.putKError(r.ErrorCode)
+ pe.putDurationMs(r.ThrottleTime)
+
+ if err := pe.putArrayLength(len(r.Errors)); err != nil {
+ return err
+ }
+ for topic, partitions := range r.Errors {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putArrayLength(len(partitions)); err != nil {
+ return err
+ }
+ for partition, errorCode := range partitions {
+ pe.putInt32(partition)
+ pe.putKError(errorCode)
+ }
+ }
+ return nil
+}
+
+func (r *DeleteOffsetsResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.ErrorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if r.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ numTopics, err := pd.getArrayLength()
+ if err != nil || numTopics == 0 {
+ return err
+ }
+
+ r.Errors = make(map[string]map[int32]KError, numTopics)
+ for i := 0; i < numTopics; i++ {
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ numErrors, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Errors[name] = make(map[int32]KError, numErrors)
+
+ for j := 0; j < numErrors; j++ {
+ id, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ r.Errors[name][id], err = pd.getKError()
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func (r *DeleteOffsetsResponse) key() int16 {
+ return apiKeyOffsetDelete
+}
+
+func (r *DeleteOffsetsResponse) version() int16 {
+ return r.Version
+}
+
+func (r *DeleteOffsetsResponse) headerVersion() int16 {
+ return 0
+}
+
+func (r *DeleteOffsetsResponse) isValidVersion() bool {
+ return r.Version == 0
+}
+
+func (r *DeleteOffsetsResponse) requiredVersion() KafkaVersion {
+ return V2_4_0_0
+}
+
+func (r *DeleteOffsetsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/delete_records_request.go b/vendor/github.com/IBM/sarama/delete_records_request.go
new file mode 100644
index 0000000..05538dd
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/delete_records_request.go
@@ -0,0 +1,145 @@
+package sarama
+
+import (
+ "slices"
+ "sort"
+ "time"
+)
+
+// request message format is:
+// [topic] timeout(int32)
+// where topic is:
+// name(string) [partition]
+// where partition is:
+// id(int32) offset(int64)
+
+type DeleteRecordsRequest struct {
+ Version int16
+ Topics map[string]*DeleteRecordsRequestTopic
+ Timeout time.Duration
+}
+
+func (d *DeleteRecordsRequest) setVersion(v int16) {
+ d.Version = v
+}
+
+func (d *DeleteRecordsRequest) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(d.Topics)); err != nil {
+ return err
+ }
+ keys := make([]string, 0, len(d.Topics))
+ for topic := range d.Topics {
+ keys = append(keys, topic)
+ }
+ sort.Strings(keys)
+ for _, topic := range keys {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := d.Topics[topic].encode(pe); err != nil {
+ return err
+ }
+ }
+ pe.putInt32(int32(d.Timeout / time.Millisecond))
+
+ return nil
+}
+
+func (d *DeleteRecordsRequest) decode(pd packetDecoder, version int16) error {
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if n > 0 {
+ d.Topics = make(map[string]*DeleteRecordsRequestTopic, n)
+ for i := 0; i < n; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ details := new(DeleteRecordsRequestTopic)
+ if err = details.decode(pd, version); err != nil {
+ return err
+ }
+ d.Topics[topic] = details
+ }
+ }
+
+ timeout, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ d.Timeout = time.Duration(timeout) * time.Millisecond
+
+ return nil
+}
+
+func (d *DeleteRecordsRequest) key() int16 {
+ return apiKeyDeleteRecords
+}
+
+func (d *DeleteRecordsRequest) version() int16 {
+ return d.Version
+}
+
+func (d *DeleteRecordsRequest) headerVersion() int16 {
+ return 1
+}
+
+func (d *DeleteRecordsRequest) isValidVersion() bool {
+ return d.Version >= 0 && d.Version <= 1
+}
+
+func (d *DeleteRecordsRequest) requiredVersion() KafkaVersion {
+ switch d.Version {
+ case 1:
+ return V2_0_0_0
+ default:
+ return V0_11_0_0
+ }
+}
+
+type DeleteRecordsRequestTopic struct {
+ PartitionOffsets map[int32]int64 // partition => offset
+}
+
+func (t *DeleteRecordsRequestTopic) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(t.PartitionOffsets)); err != nil {
+ return err
+ }
+ keys := make([]int32, 0, len(t.PartitionOffsets))
+ for partition := range t.PartitionOffsets {
+ keys = append(keys, partition)
+ }
+ slices.Sort(keys)
+ for _, partition := range keys {
+ pe.putInt32(partition)
+ pe.putInt64(t.PartitionOffsets[partition])
+ }
+ return nil
+}
+
+func (t *DeleteRecordsRequestTopic) decode(pd packetDecoder, version int16) error {
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if n > 0 {
+ t.PartitionOffsets = make(map[int32]int64, n)
+ for i := 0; i < n; i++ {
+ partition, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ offset, err := pd.getInt64()
+ if err != nil {
+ return err
+ }
+ t.PartitionOffsets[partition] = offset
+ }
+ }
+
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/delete_records_response.go b/vendor/github.com/IBM/sarama/delete_records_response.go
new file mode 100644
index 0000000..4a41811
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/delete_records_response.go
@@ -0,0 +1,177 @@
+package sarama
+
+import (
+ "slices"
+ "sort"
+ "time"
+)
+
+// response message format is:
+// throttleMs(int32) [topic]
+// where topic is:
+// name(string) [partition]
+// where partition is:
+// id(int32) low_watermark(int64) error_code(int16)
+
+type DeleteRecordsResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ Topics map[string]*DeleteRecordsResponseTopic
+}
+
+func (d *DeleteRecordsResponse) setVersion(v int16) {
+ d.Version = v
+}
+
+func (d *DeleteRecordsResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(d.ThrottleTime)
+
+ if err := pe.putArrayLength(len(d.Topics)); err != nil {
+ return err
+ }
+ keys := make([]string, 0, len(d.Topics))
+ for topic := range d.Topics {
+ keys = append(keys, topic)
+ }
+ sort.Strings(keys)
+ for _, topic := range keys {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := d.Topics[topic].encode(pe); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (d *DeleteRecordsResponse) decode(pd packetDecoder, version int16) (err error) {
+ d.Version = version
+
+ if d.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if n > 0 {
+ d.Topics = make(map[string]*DeleteRecordsResponseTopic, n)
+ for i := 0; i < n; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ details := new(DeleteRecordsResponseTopic)
+ if err = details.decode(pd, version); err != nil {
+ return err
+ }
+ d.Topics[topic] = details
+ }
+ }
+
+ return nil
+}
+
+func (d *DeleteRecordsResponse) key() int16 {
+ return apiKeyDeleteRecords
+}
+
+func (d *DeleteRecordsResponse) version() int16 {
+ return d.Version
+}
+
+func (d *DeleteRecordsResponse) headerVersion() int16 {
+ return 0
+}
+
+func (d *DeleteRecordsResponse) isValidVersion() bool {
+ return d.Version >= 0 && d.Version <= 1
+}
+
+func (d *DeleteRecordsResponse) requiredVersion() KafkaVersion {
+ switch d.Version {
+ case 1:
+ return V2_0_0_0
+ default:
+ return V0_11_0_0
+ }
+}
+
+func (r *DeleteRecordsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
+
+type DeleteRecordsResponseTopic struct {
+ Partitions map[int32]*DeleteRecordsResponsePartition
+}
+
+func (t *DeleteRecordsResponseTopic) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(t.Partitions)); err != nil {
+ return err
+ }
+ keys := make([]int32, 0, len(t.Partitions))
+ for partition := range t.Partitions {
+ keys = append(keys, partition)
+ }
+ slices.Sort(keys)
+ for _, partition := range keys {
+ pe.putInt32(partition)
+ if err := t.Partitions[partition].encode(pe); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (t *DeleteRecordsResponseTopic) decode(pd packetDecoder, version int16) error {
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if n > 0 {
+ t.Partitions = make(map[int32]*DeleteRecordsResponsePartition, n)
+ for i := 0; i < n; i++ {
+ partition, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ details := new(DeleteRecordsResponsePartition)
+ if err = details.decode(pd, version); err != nil {
+ return err
+ }
+ t.Partitions[partition] = details
+ }
+ }
+
+ return nil
+}
+
+type DeleteRecordsResponsePartition struct {
+ LowWatermark int64
+ Err KError
+}
+
+func (t *DeleteRecordsResponsePartition) encode(pe packetEncoder) error {
+ pe.putInt64(t.LowWatermark)
+ pe.putKError(t.Err)
+ return nil
+}
+
+func (t *DeleteRecordsResponsePartition) decode(pd packetDecoder, version int16) error {
+ lowWatermark, err := pd.getInt64()
+ if err != nil {
+ return err
+ }
+ t.LowWatermark = lowWatermark
+
+ t.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/delete_topics_request.go b/vendor/github.com/IBM/sarama/delete_topics_request.go
new file mode 100644
index 0000000..cb9e924
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/delete_topics_request.go
@@ -0,0 +1,98 @@
+package sarama
+
+import "time"
+
+type DeleteTopicsRequest struct {
+ Version int16
+ Topics []string
+ Timeout time.Duration
+}
+
+func (d *DeleteTopicsRequest) setVersion(v int16) {
+ d.Version = v
+}
+
+func NewDeleteTopicsRequest(version KafkaVersion, topics []string, timeout time.Duration) *DeleteTopicsRequest {
+ d := &DeleteTopicsRequest{
+ Topics: topics,
+ Timeout: timeout,
+ }
+ if version.IsAtLeast(V2_4_0_0) {
+ d.Version = 4
+ } else if version.IsAtLeast(V2_1_0_0) {
+ d.Version = 3
+ } else if version.IsAtLeast(V2_0_0_0) {
+ d.Version = 2
+ } else if version.IsAtLeast(V0_11_0_0) {
+ d.Version = 1
+ }
+ return d
+}
+
+func (d *DeleteTopicsRequest) encode(pe packetEncoder) error {
+ if err := pe.putStringArray(d.Topics); err != nil {
+ return err
+ }
+ pe.putInt32(int32(d.Timeout / time.Millisecond))
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (d *DeleteTopicsRequest) decode(pd packetDecoder, version int16) (err error) {
+ if d.Topics, err = pd.getStringArray(); err != nil {
+ return err
+ }
+ timeout, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ d.Timeout = time.Duration(timeout) * time.Millisecond
+ d.Version = version
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (d *DeleteTopicsRequest) key() int16 {
+ return apiKeyDeleteTopics
+}
+
+func (d *DeleteTopicsRequest) version() int16 {
+ return d.Version
+}
+
+func (d *DeleteTopicsRequest) headerVersion() int16 {
+ if d.Version >= 4 {
+ return 2
+ }
+ return 1
+}
+
+func (d *DeleteTopicsRequest) isFlexible() bool {
+ return d.isFlexibleVersion(d.Version)
+}
+
+func (d *DeleteTopicsRequest) isFlexibleVersion(version int16) bool {
+ return version >= 4
+}
+
+func (d *DeleteTopicsRequest) isValidVersion() bool {
+ return d.Version >= 0 && d.Version <= 4
+}
+
+func (d *DeleteTopicsRequest) requiredVersion() KafkaVersion {
+ switch d.Version {
+ case 4:
+ return V2_4_0_0
+ case 3:
+ return V2_1_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_10_1_0
+ default:
+ return V2_2_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/delete_topics_response.go b/vendor/github.com/IBM/sarama/delete_topics_response.go
new file mode 100644
index 0000000..4b53f0b
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/delete_topics_response.go
@@ -0,0 +1,118 @@
+package sarama
+
+import (
+ "time"
+)
+
+type DeleteTopicsResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ TopicErrorCodes map[string]KError
+}
+
+func (d *DeleteTopicsResponse) setVersion(v int16) {
+ d.Version = v
+}
+
+func (d *DeleteTopicsResponse) encode(pe packetEncoder) error {
+ if d.Version >= 1 {
+ pe.putDurationMs(d.ThrottleTime)
+ }
+
+ if err := pe.putArrayLength(len(d.TopicErrorCodes)); err != nil {
+ return err
+ }
+ for topic, errorCode := range d.TopicErrorCodes {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ pe.putKError(errorCode)
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (d *DeleteTopicsResponse) decode(pd packetDecoder, version int16) (err error) {
+ if version >= 1 {
+ if d.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ d.Version = version
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ d.TopicErrorCodes = make(map[string]KError, n)
+
+ for i := 0; i < n; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ d.TopicErrorCodes[topic], err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (d *DeleteTopicsResponse) key() int16 {
+ return apiKeyDeleteTopics
+}
+
+func (d *DeleteTopicsResponse) version() int16 {
+ return d.Version
+}
+
+func (d *DeleteTopicsResponse) headerVersion() int16 {
+ if d.Version >= 4 {
+ return 1
+ }
+ return 0
+}
+
+func (d *DeleteTopicsResponse) isFlexible() bool {
+ return d.isFlexibleVersion(d.Version)
+}
+
+func (d *DeleteTopicsResponse) isFlexibleVersion(version int16) bool {
+ return version >= 4
+}
+
+func (d *DeleteTopicsResponse) isValidVersion() bool {
+ return d.Version >= 0 && d.Version <= 4
+}
+
+func (d *DeleteTopicsResponse) requiredVersion() KafkaVersion {
+ switch d.Version {
+ case 4:
+ return V2_4_0_0
+ case 3:
+ return V2_1_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_10_1_0
+ default:
+ return V2_2_0_0
+ }
+}
+
+func (r *DeleteTopicsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/describe_client_quotas_request.go b/vendor/github.com/IBM/sarama/describe_client_quotas_request.go
new file mode 100644
index 0000000..120ed33
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/describe_client_quotas_request.go
@@ -0,0 +1,194 @@
+package sarama
+
+// DescribeClientQuotas Request (Version: 0) => [components] strict
+// components => entity_type match_type match
+// entity_type => STRING
+// match_type => INT8
+// match => NULLABLE_STRING
+// strict => BOOLEAN
+// DescribeClientQuotas Request (Version: 1) => [components] strict _tagged_fields
+// components => entity_type match_type match _tagged_fields
+// entity_type => COMPACT_STRING
+// match_type => INT8
+// match => COMPACT_NULLABLE_STRING
+// strict => BOOLEAN
+
+// DescribeClientQuotasRequest contains a filter to be applied to matching
+// client quotas.
+// Components: the components to filter on
+// Strict: whether the filter only includes specified components
+type DescribeClientQuotasRequest struct {
+ Version int16
+ Components []QuotaFilterComponent
+ Strict bool
+}
+
+func NewDescribeClientQuotasRequest(version KafkaVersion, components []QuotaFilterComponent, strict bool) *DescribeClientQuotasRequest {
+ d := &DescribeClientQuotasRequest{
+ Components: components,
+ Strict: strict,
+ }
+ if version.IsAtLeast(V2_8_0_0) {
+ d.Version = 1
+ }
+ return d
+}
+
+func (d *DescribeClientQuotasRequest) setVersion(v int16) {
+ d.Version = v
+}
+
+// QuotaFilterComponent describes a component for applying a client quota filter.
+// EntityType: the entity type the filter component applies to ("user", "client-id", "ip")
+// MatchType: the match type of the filter component (any, exact, default)
+// Match: the name that's matched exactly (used when MatchType is QuotaMatchExact)
+type QuotaFilterComponent struct {
+ EntityType QuotaEntityType
+ MatchType QuotaMatchType
+ Match string
+}
+
+func (d *DescribeClientQuotasRequest) encode(pe packetEncoder) error {
+ // Components
+ if err := pe.putArrayLength(len(d.Components)); err != nil {
+ return err
+ }
+ for _, c := range d.Components {
+ if err := c.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ // Strict
+ pe.putBool(d.Strict)
+
+ pe.putEmptyTaggedFieldArray()
+
+ return nil
+}
+
+func (d *DescribeClientQuotasRequest) decode(pd packetDecoder, version int16) error {
+ // Components
+ componentCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if componentCount > 0 {
+ d.Components = make([]QuotaFilterComponent, componentCount)
+ for i := range d.Components {
+ c := QuotaFilterComponent{}
+ if err = c.decode(pd, version); err != nil {
+ return err
+ }
+ d.Components[i] = c
+ }
+ } else {
+ d.Components = []QuotaFilterComponent{}
+ }
+
+ // Strict
+ strict, err := pd.getBool()
+ if err != nil {
+ return err
+ }
+ d.Strict = strict
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (d *QuotaFilterComponent) encode(pe packetEncoder) error {
+ // EntityType
+ if err := pe.putString(string(d.EntityType)); err != nil {
+ return err
+ }
+
+ // MatchType
+ pe.putInt8(int8(d.MatchType))
+
+ // Match
+ if d.MatchType == QuotaMatchAny {
+ if err := pe.putNullableString(nil); err != nil {
+ return err
+ }
+ } else if d.MatchType == QuotaMatchDefault {
+ if err := pe.putNullableString(nil); err != nil {
+ return err
+ }
+ } else {
+ if err := pe.putString(d.Match); err != nil {
+ return err
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+
+ return nil
+}
+
+func (d *QuotaFilterComponent) decode(pd packetDecoder, version int16) error {
+ // EntityType
+ entityType, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ d.EntityType = QuotaEntityType(entityType)
+
+ // MatchType
+ matchType, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ d.MatchType = QuotaMatchType(matchType)
+
+ // Match
+ match, err := pd.getNullableString()
+ if err != nil {
+ return err
+ }
+ if match != nil {
+ d.Match = *match
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (d *DescribeClientQuotasRequest) key() int16 {
+ return apiKeyDescribeClientQuotas
+}
+
+func (d *DescribeClientQuotasRequest) version() int16 {
+ return d.Version
+}
+
+func (d *DescribeClientQuotasRequest) headerVersion() int16 {
+ if d.Version >= 1 {
+ return 2
+ }
+
+ return 1
+}
+
+func (d *DescribeClientQuotasRequest) isValidVersion() bool {
+ return d.Version >= 0 && d.Version <= 1
+}
+
+func (d *DescribeClientQuotasRequest) isFlexible() bool {
+ return d.isFlexibleVersion(d.Version)
+}
+
+func (d *DescribeClientQuotasRequest) isFlexibleVersion(version int16) bool {
+ return version >= 1
+}
+
+func (d *DescribeClientQuotasRequest) requiredVersion() KafkaVersion {
+ switch d.Version {
+ case 1:
+ return V2_8_0_0
+ case 0:
+ return V2_6_0_0
+ default:
+ return V2_8_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/describe_client_quotas_response.go b/vendor/github.com/IBM/sarama/describe_client_quotas_response.go
new file mode 100644
index 0000000..4cf9e5b
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/describe_client_quotas_response.go
@@ -0,0 +1,285 @@
+package sarama
+
+import (
+ "time"
+)
+
+// DescribeClientQuotas Response (Version: 0) => throttle_time_ms error_code error_message [entries]
+// throttle_time_ms => INT32
+// error_code => INT16
+// error_message => NULLABLE_STRING
+// entries => [entity] [values]
+// entity => entity_type entity_name
+// entity_type => STRING
+// entity_name => NULLABLE_STRING
+// values => key value
+// key => STRING
+// value => FLOAT64
+// DescribeClientQuotas Response (Version: 1) => throttle_time_ms error_code error_message [entries] _tagged_fields
+// throttle_time_ms => INT32
+// error_code => INT16
+// error_message => COMPACT_NULLABLE_STRING
+// entries => [entity] [values] _tagged_fields
+// entity => entity_type entity_name _tagged_fields
+// entity_type => COMPACT_STRING
+// entity_name => COMPACT_NULLABLE_STRING
+// values => key value _tagged_fields
+// key => COMPACT_STRING
+// value => FLOAT64
+
+type DescribeClientQuotasResponse struct {
+ Version int16
+ ThrottleTime time.Duration // The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota.
+ ErrorCode KError // The error code, or `0` if the quota description succeeded.
+ ErrorMsg *string // The error message, or `null` if the quota description succeeded.
+ Entries []DescribeClientQuotasEntry // A result entry.
+}
+
+func (d *DescribeClientQuotasResponse) setVersion(v int16) {
+ d.Version = v
+}
+
+type DescribeClientQuotasEntry struct {
+ Entity []QuotaEntityComponent // The quota entity description.
+ Values map[string]float64 // The quota values for the entity.
+}
+
+type QuotaEntityComponent struct {
+ EntityType QuotaEntityType
+ MatchType QuotaMatchType
+ Name string
+}
+
+func (d *DescribeClientQuotasResponse) encode(pe packetEncoder) error {
+ // ThrottleTime
+ pe.putDurationMs(d.ThrottleTime)
+
+ // ErrorCode
+ pe.putKError(d.ErrorCode)
+
+ // ErrorMsg
+ if err := pe.putNullableString(d.ErrorMsg); err != nil {
+ return err
+ }
+
+ // Entries
+ if err := pe.putArrayLength(len(d.Entries)); err != nil {
+ return err
+ }
+ for _, e := range d.Entries {
+ if err := e.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (d *DescribeClientQuotasResponse) decode(pd packetDecoder, version int16) (err error) {
+ if d.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ // ErrorCode
+ d.ErrorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ // ErrorMsg
+ errMsg, err := pd.getNullableString()
+ if err != nil {
+ return err
+ }
+ d.ErrorMsg = errMsg
+
+ // Entries
+ entryCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if entryCount > 0 {
+ d.Entries = make([]DescribeClientQuotasEntry, entryCount)
+ for i := range d.Entries {
+ e := DescribeClientQuotasEntry{}
+ if err = e.decode(pd, version); err != nil {
+ return err
+ }
+ d.Entries[i] = e
+ }
+ } else {
+ d.Entries = []DescribeClientQuotasEntry{}
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (d *DescribeClientQuotasEntry) encode(pe packetEncoder) error {
+ // Entity
+ if err := pe.putArrayLength(len(d.Entity)); err != nil {
+ return err
+ }
+ for _, e := range d.Entity {
+ if err := e.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ // Values
+ if err := pe.putArrayLength(len(d.Values)); err != nil {
+ return err
+ }
+ for key, value := range d.Values {
+ // key
+ if err := pe.putString(key); err != nil {
+ return err
+ }
+ // value
+ pe.putFloat64(value)
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (d *DescribeClientQuotasEntry) decode(pd packetDecoder, version int16) error {
+ // Entity
+ componentCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if componentCount > 0 {
+ d.Entity = make([]QuotaEntityComponent, componentCount)
+ for i := 0; i < componentCount; i++ {
+ component := QuotaEntityComponent{}
+ if err := component.decode(pd, version); err != nil {
+ return err
+ }
+ d.Entity[i] = component
+ }
+ } else {
+ d.Entity = []QuotaEntityComponent{}
+ }
+
+ // Values
+ valueCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if valueCount > 0 {
+ d.Values = make(map[string]float64, valueCount)
+ for i := 0; i < valueCount; i++ {
+ // key
+ key, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ // value
+ value, err := pd.getFloat64()
+ if err != nil {
+ return err
+ }
+ d.Values[key] = value
+ _, err = pd.getEmptyTaggedFieldArray()
+ if err != nil {
+ return err
+ }
+ }
+ } else {
+ d.Values = map[string]float64{}
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (c *QuotaEntityComponent) encode(pe packetEncoder) error {
+ // entity_type
+ if err := pe.putString(string(c.EntityType)); err != nil {
+ return err
+ }
+ // entity_name
+ if c.MatchType == QuotaMatchDefault {
+ if err := pe.putNullableString(nil); err != nil {
+ return err
+ }
+ } else {
+ if err := pe.putString(c.Name); err != nil {
+ return err
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (c *QuotaEntityComponent) decode(pd packetDecoder, version int16) error {
+ // entity_type
+ entityType, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ c.EntityType = QuotaEntityType(entityType)
+
+ // entity_name
+ entityName, err := pd.getNullableString()
+ if err != nil {
+ return err
+ }
+
+ if entityName == nil {
+ c.MatchType = QuotaMatchDefault
+ } else {
+ c.MatchType = QuotaMatchExact
+ c.Name = *entityName
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (d *DescribeClientQuotasResponse) key() int16 {
+ return apiKeyDescribeClientQuotas
+}
+
+func (d *DescribeClientQuotasResponse) version() int16 {
+ return d.Version
+}
+
+func (d *DescribeClientQuotasResponse) headerVersion() int16 {
+ if d.Version >= 1 {
+ return 1
+ }
+
+ return 0
+}
+
+func (d *DescribeClientQuotasResponse) isValidVersion() bool {
+ return d.Version >= 0 && d.Version <= 1
+}
+
+func (d *DescribeClientQuotasResponse) isFlexible() bool {
+ return d.isFlexibleVersion(d.Version)
+}
+
+func (d *DescribeClientQuotasResponse) isFlexibleVersion(version int16) bool {
+ return version >= 1
+}
+
+func (d *DescribeClientQuotasResponse) requiredVersion() KafkaVersion {
+ switch d.Version {
+ case 1:
+ return V2_8_0_0
+ case 0:
+ return V2_6_0_0
+ default:
+ return V2_8_0_0
+ }
+}
+
+func (r *DescribeClientQuotasResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/describe_configs_request.go b/vendor/github.com/IBM/sarama/describe_configs_request.go
new file mode 100644
index 0000000..d42e271
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/describe_configs_request.go
@@ -0,0 +1,125 @@
+package sarama
+
+type DescribeConfigsRequest struct {
+ Version int16
+ Resources []*ConfigResource
+ IncludeSynonyms bool
+}
+
+func (r *DescribeConfigsRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+type ConfigResource struct {
+ Type ConfigResourceType
+ Name string
+ ConfigNames []string
+}
+
+func (r *DescribeConfigsRequest) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(r.Resources)); err != nil {
+ return err
+ }
+
+ for _, c := range r.Resources {
+ pe.putInt8(int8(c.Type))
+ if err := pe.putString(c.Name); err != nil {
+ return err
+ }
+
+ if len(c.ConfigNames) == 0 {
+ pe.putInt32(-1)
+ continue
+ }
+ if err := pe.putStringArray(c.ConfigNames); err != nil {
+ return err
+ }
+ }
+
+ if r.Version >= 1 {
+ pe.putBool(r.IncludeSynonyms)
+ }
+
+ return nil
+}
+
+func (r *DescribeConfigsRequest) decode(pd packetDecoder, version int16) (err error) {
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Resources = make([]*ConfigResource, n)
+
+ for i := 0; i < n; i++ {
+ r.Resources[i] = &ConfigResource{}
+ t, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ r.Resources[i].Type = ConfigResourceType(t)
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ r.Resources[i].Name = name
+
+ confLength, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if confLength == -1 {
+ continue
+ }
+
+ cfnames := make([]string, confLength)
+ for i := 0; i < confLength; i++ {
+ s, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ cfnames[i] = s
+ }
+ r.Resources[i].ConfigNames = cfnames
+ }
+ r.Version = version
+ if r.Version >= 1 {
+ b, err := pd.getBool()
+ if err != nil {
+ return err
+ }
+ r.IncludeSynonyms = b
+ }
+
+ return nil
+}
+
+func (r *DescribeConfigsRequest) key() int16 {
+ return apiKeyDescribeConfigs
+}
+
+func (r *DescribeConfigsRequest) version() int16 {
+ return r.Version
+}
+
+func (r *DescribeConfigsRequest) headerVersion() int16 {
+ return 1
+}
+
+func (r *DescribeConfigsRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 2
+}
+
+func (r *DescribeConfigsRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V1_1_0_0
+ case 0:
+ return V0_11_0_0
+ default:
+ return V2_0_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/describe_configs_response.go b/vendor/github.com/IBM/sarama/describe_configs_response.go
new file mode 100644
index 0000000..c702f02
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/describe_configs_response.go
@@ -0,0 +1,352 @@
+package sarama
+
+import (
+ "fmt"
+ "time"
+)
+
+type ConfigSource int8
+
+func (s ConfigSource) String() string {
+ switch s {
+ case SourceUnknown:
+ return "Unknown"
+ case SourceTopic:
+ return "Topic"
+ case SourceDynamicBroker:
+ return "DynamicBroker"
+ case SourceDynamicDefaultBroker:
+ return "DynamicDefaultBroker"
+ case SourceStaticBroker:
+ return "StaticBroker"
+ case SourceDefault:
+ return "Default"
+ }
+ return fmt.Sprintf("Source Invalid: %d", int(s))
+}
+
+const (
+ SourceUnknown ConfigSource = iota
+ SourceTopic
+ SourceDynamicBroker
+ SourceDynamicDefaultBroker
+ SourceStaticBroker
+ SourceDefault
+)
+
+type DescribeConfigError struct {
+ Err KError
+ ErrMsg string
+}
+
+func (c *DescribeConfigError) Error() string {
+ text := c.Err.Error()
+ if c.ErrMsg != "" {
+ text = fmt.Sprintf("%s - %s", text, c.ErrMsg)
+ }
+ return text
+}
+
+type DescribeConfigsResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ Resources []*ResourceResponse
+}
+
+func (r *DescribeConfigsResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+type ResourceResponse struct {
+ ErrorCode int16
+ ErrorMsg string
+ Type ConfigResourceType
+ Name string
+ Configs []*ConfigEntry
+}
+
+type ConfigEntry struct {
+ Name string
+ Value string
+ ReadOnly bool
+ Default bool
+ Source ConfigSource
+ Sensitive bool
+ Synonyms []*ConfigSynonym
+}
+
+type ConfigSynonym struct {
+ ConfigName string
+ ConfigValue string
+ Source ConfigSource
+}
+
+func (r *DescribeConfigsResponse) encode(pe packetEncoder) (err error) {
+ pe.putDurationMs(r.ThrottleTime)
+ if err = pe.putArrayLength(len(r.Resources)); err != nil {
+ return err
+ }
+
+ for _, c := range r.Resources {
+ if err = c.encode(pe, r.Version); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (r *DescribeConfigsResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ if r.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Resources = make([]*ResourceResponse, n)
+ for i := 0; i < n; i++ {
+ rr := &ResourceResponse{}
+ if err := rr.decode(pd, version); err != nil {
+ return err
+ }
+ r.Resources[i] = rr
+ }
+
+ return nil
+}
+
+func (r *DescribeConfigsResponse) key() int16 {
+ return apiKeyDescribeConfigs
+}
+
+func (r *DescribeConfigsResponse) version() int16 {
+ return r.Version
+}
+
+func (r *DescribeConfigsResponse) headerVersion() int16 {
+ return 0
+}
+
+func (r *DescribeConfigsResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 2
+}
+
+func (r *DescribeConfigsResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V1_1_0_0
+ case 0:
+ return V0_11_0_0
+ default:
+ return V2_0_0_0
+ }
+}
+
+func (r *DescribeConfigsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
+
+func (r *ResourceResponse) encode(pe packetEncoder, version int16) (err error) {
+ pe.putInt16(r.ErrorCode)
+
+ if err = pe.putString(r.ErrorMsg); err != nil {
+ return err
+ }
+
+ pe.putInt8(int8(r.Type))
+
+ if err = pe.putString(r.Name); err != nil {
+ return err
+ }
+
+ if err = pe.putArrayLength(len(r.Configs)); err != nil {
+ return err
+ }
+
+ for _, c := range r.Configs {
+ if err = c.encode(pe, version); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (r *ResourceResponse) decode(pd packetDecoder, version int16) (err error) {
+ ec, err := pd.getInt16()
+ if err != nil {
+ return err
+ }
+ r.ErrorCode = ec
+
+ em, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ r.ErrorMsg = em
+
+ t, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ r.Type = ConfigResourceType(t)
+
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ r.Name = name
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Configs = make([]*ConfigEntry, n)
+ for i := 0; i < n; i++ {
+ c := &ConfigEntry{}
+ if err := c.decode(pd, version); err != nil {
+ return err
+ }
+ r.Configs[i] = c
+ }
+ return nil
+}
+
+func (r *ConfigEntry) encode(pe packetEncoder, version int16) (err error) {
+ if err = pe.putString(r.Name); err != nil {
+ return err
+ }
+
+ if err = pe.putString(r.Value); err != nil {
+ return err
+ }
+
+ pe.putBool(r.ReadOnly)
+
+ if version <= 0 {
+ pe.putBool(r.Default)
+ pe.putBool(r.Sensitive)
+ } else {
+ pe.putInt8(int8(r.Source))
+ pe.putBool(r.Sensitive)
+
+ if err := pe.putArrayLength(len(r.Synonyms)); err != nil {
+ return err
+ }
+ for _, c := range r.Synonyms {
+ if err = c.encode(pe, version); err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+// https://cwiki.apache.org/confluence/display/KAFKA/KIP-226+-+Dynamic+Broker+Configuration
+func (r *ConfigEntry) decode(pd packetDecoder, version int16) (err error) {
+ if version == 0 {
+ r.Source = SourceUnknown
+ }
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ r.Name = name
+
+ value, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ r.Value = value
+
+ read, err := pd.getBool()
+ if err != nil {
+ return err
+ }
+ r.ReadOnly = read
+
+ if version == 0 {
+ defaultB, err := pd.getBool()
+ if err != nil {
+ return err
+ }
+ r.Default = defaultB
+ if defaultB {
+ r.Source = SourceDefault
+ }
+ } else {
+ source, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ r.Source = ConfigSource(source)
+ r.Default = r.Source == SourceDefault
+ }
+
+ sensitive, err := pd.getBool()
+ if err != nil {
+ return err
+ }
+ r.Sensitive = sensitive
+
+ if version > 0 {
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ r.Synonyms = make([]*ConfigSynonym, n)
+
+ for i := 0; i < n; i++ {
+ s := &ConfigSynonym{}
+ if err := s.decode(pd, version); err != nil {
+ return err
+ }
+ r.Synonyms[i] = s
+ }
+ }
+ return nil
+}
+
+func (c *ConfigSynonym) encode(pe packetEncoder, version int16) (err error) {
+ err = pe.putString(c.ConfigName)
+ if err != nil {
+ return err
+ }
+
+ err = pe.putString(c.ConfigValue)
+ if err != nil {
+ return err
+ }
+
+ pe.putInt8(int8(c.Source))
+
+ return nil
+}
+
+func (c *ConfigSynonym) decode(pd packetDecoder, version int16) error {
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ c.ConfigName = name
+
+ value, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ c.ConfigValue = value
+
+ source, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ c.Source = ConfigSource(source)
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/describe_groups_request.go b/vendor/github.com/IBM/sarama/describe_groups_request.go
new file mode 100644
index 0000000..b2fb07e
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/describe_groups_request.go
@@ -0,0 +1,87 @@
+package sarama
+
+type DescribeGroupsRequest struct {
+ Version int16
+ Groups []string
+ IncludeAuthorizedOperations bool
+}
+
+func (r *DescribeGroupsRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *DescribeGroupsRequest) encode(pe packetEncoder) error {
+ if err := pe.putStringArray(r.Groups); err != nil {
+ return err
+ }
+ if r.Version >= 3 {
+ pe.putBool(r.IncludeAuthorizedOperations)
+ }
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *DescribeGroupsRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ r.Groups, err = pd.getStringArray()
+ if err != nil {
+ return err
+ }
+ if r.Version >= 3 {
+ if r.IncludeAuthorizedOperations, err = pd.getBool(); err != nil {
+ return err
+ }
+ }
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *DescribeGroupsRequest) key() int16 {
+ return apiKeyDescribeGroups
+}
+
+func (r *DescribeGroupsRequest) version() int16 {
+ return r.Version
+}
+
+func (r *DescribeGroupsRequest) headerVersion() int16 {
+ if r.Version >= 5 {
+ return 2
+ }
+ return 1
+}
+
+func (r *DescribeGroupsRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 5
+}
+
+func (r *DescribeGroupsRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *DescribeGroupsRequest) isFlexibleVersion(version int16) bool {
+ return version >= 5
+}
+
+func (r *DescribeGroupsRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 5:
+ return V2_4_0_0
+ case 4:
+ return V2_4_0_0
+ case 3:
+ return V2_3_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_9_0_0
+ default:
+ return V2_4_0_0
+ }
+}
+
+func (r *DescribeGroupsRequest) AddGroup(group string) {
+ r.Groups = append(r.Groups, group)
+}
diff --git a/vendor/github.com/IBM/sarama/describe_groups_response.go b/vendor/github.com/IBM/sarama/describe_groups_response.go
new file mode 100644
index 0000000..dcc274d
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/describe_groups_response.go
@@ -0,0 +1,306 @@
+package sarama
+
+import "time"
+
+type DescribeGroupsResponse struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // ThrottleTimeMs contains the duration in milliseconds for which the
+ // request was throttled due to a quota violation, or zero if the request
+ // did not violate any quota.
+ ThrottleTimeMs int32
+ // Groups contains each described group.
+ Groups []*GroupDescription
+}
+
+func (r *DescribeGroupsResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *DescribeGroupsResponse) encode(pe packetEncoder) (err error) {
+ if r.Version >= 1 {
+ pe.putInt32(r.ThrottleTimeMs)
+ }
+ if err := pe.putArrayLength(len(r.Groups)); err != nil {
+ return err
+ }
+
+ for _, block := range r.Groups {
+ if err := block.encode(pe, r.Version); err != nil {
+ return err
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *DescribeGroupsResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ if r.Version >= 1 {
+ if r.ThrottleTimeMs, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+ if numGroups, err := pd.getArrayLength(); err != nil {
+ return err
+ } else if numGroups > 0 {
+ r.Groups = make([]*GroupDescription, numGroups)
+ for i := 0; i < numGroups; i++ {
+ block := &GroupDescription{}
+ if err := block.decode(pd, r.Version); err != nil {
+ return err
+ }
+ r.Groups[i] = block
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *DescribeGroupsResponse) key() int16 {
+ return apiKeyDescribeGroups
+}
+
+func (r *DescribeGroupsResponse) version() int16 {
+ return r.Version
+}
+
+func (r *DescribeGroupsResponse) headerVersion() int16 {
+ if r.Version >= 5 {
+ return 1
+ }
+ return 0
+}
+
+func (r *DescribeGroupsResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 5
+}
+
+func (r *DescribeGroupsResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *DescribeGroupsResponse) isFlexibleVersion(version int16) bool {
+ return version >= 5
+}
+
+func (r *DescribeGroupsResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 5:
+ return V2_4_0_0
+ case 4:
+ return V2_4_0_0
+ case 3:
+ return V2_3_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_9_0_0
+ default:
+ return V2_4_0_0
+ }
+}
+
+func (r *DescribeGroupsResponse) throttleTime() time.Duration {
+ return time.Duration(r.ThrottleTimeMs) * time.Millisecond
+}
+
+// GroupDescription contains each described group.
+type GroupDescription struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // Err contains the describe error as the KError type.
+ Err KError
+ // ErrorCode contains the describe error, or 0 if there was no error.
+ ErrorCode int16
+ // GroupId contains the group ID string.
+ GroupId string
+ // State contains the group state string, or the empty string.
+ State string
+ // ProtocolType contains the group protocol type, or the empty string.
+ ProtocolType string
+ // Protocol contains the group protocol data, or the empty string.
+ Protocol string
+ // Members contains the group members.
+ Members map[string]*GroupMemberDescription
+ // AuthorizedOperations contains a 32-bit bitfield to represent authorized
+ // operations for this group.
+ AuthorizedOperations int32
+}
+
+func (gd *GroupDescription) encode(pe packetEncoder, version int16) (err error) {
+ gd.Version = version
+ pe.putInt16(gd.ErrorCode)
+
+ if err := pe.putString(gd.GroupId); err != nil {
+ return err
+ }
+ if err := pe.putString(gd.State); err != nil {
+ return err
+ }
+ if err := pe.putString(gd.ProtocolType); err != nil {
+ return err
+ }
+ if err := pe.putString(gd.Protocol); err != nil {
+ return err
+ }
+
+ if err := pe.putArrayLength(len(gd.Members)); err != nil {
+ return err
+ }
+
+ for _, block := range gd.Members {
+ if err := block.encode(pe, gd.Version); err != nil {
+ return err
+ }
+ }
+
+ if gd.Version >= 3 {
+ pe.putInt32(gd.AuthorizedOperations)
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (gd *GroupDescription) decode(pd packetDecoder, version int16) (err error) {
+ gd.Version = version
+ if gd.ErrorCode, err = pd.getInt16(); err != nil {
+ return err
+ }
+
+ gd.Err = KError(gd.ErrorCode)
+
+ if gd.GroupId, err = pd.getString(); err != nil {
+ return err
+ }
+ if gd.State, err = pd.getString(); err != nil {
+ return err
+ }
+ if gd.ProtocolType, err = pd.getString(); err != nil {
+ return err
+ }
+ if gd.Protocol, err = pd.getString(); err != nil {
+ return err
+ }
+
+ if numMembers, err := pd.getArrayLength(); err != nil {
+ return err
+ } else if numMembers > 0 {
+ gd.Members = make(map[string]*GroupMemberDescription, numMembers)
+ for i := 0; i < numMembers; i++ {
+ block := &GroupMemberDescription{}
+ if err := block.decode(pd, gd.Version); err != nil {
+ return err
+ }
+ gd.Members[block.MemberId] = block
+ }
+ }
+
+ if gd.Version >= 3 {
+ if gd.AuthorizedOperations, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+// GroupMemberDescription contains the group members.
+type GroupMemberDescription struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // MemberId contains the member ID assigned by the group coordinator.
+ MemberId string
+ // GroupInstanceId contains the unique identifier of the consumer instance
+ // provided by end user.
+ GroupInstanceId *string
+ // ClientId contains the client ID used in the member's latest join group
+ // request.
+ ClientId string
+ // ClientHost contains the client host.
+ ClientHost string
+ // MemberMetadata contains the metadata corresponding to the current group
+ // protocol in use.
+ MemberMetadata []byte
+ // MemberAssignment contains the current assignment provided by the group
+ // leader.
+ MemberAssignment []byte
+}
+
+func (gmd *GroupMemberDescription) encode(pe packetEncoder, version int16) (err error) {
+ gmd.Version = version
+ if err := pe.putString(gmd.MemberId); err != nil {
+ return err
+ }
+ if gmd.Version >= 4 {
+ if err := pe.putNullableString(gmd.GroupInstanceId); err != nil {
+ return err
+ }
+ }
+ if err := pe.putString(gmd.ClientId); err != nil {
+ return err
+ }
+ if err := pe.putString(gmd.ClientHost); err != nil {
+ return err
+ }
+ if err := pe.putBytes(gmd.MemberMetadata); err != nil {
+ return err
+ }
+ if err := pe.putBytes(gmd.MemberAssignment); err != nil {
+ return err
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (gmd *GroupMemberDescription) decode(pd packetDecoder, version int16) (err error) {
+ gmd.Version = version
+ if gmd.MemberId, err = pd.getString(); err != nil {
+ return err
+ }
+ if gmd.Version >= 4 {
+ if gmd.GroupInstanceId, err = pd.getNullableString(); err != nil {
+ return err
+ }
+ }
+ if gmd.ClientId, err = pd.getString(); err != nil {
+ return err
+ }
+ if gmd.ClientHost, err = pd.getString(); err != nil {
+ return err
+ }
+ if gmd.MemberMetadata, err = pd.getBytes(); err != nil {
+ return err
+ }
+ if gmd.MemberAssignment, err = pd.getBytes(); err != nil {
+ return err
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (gmd *GroupMemberDescription) GetMemberAssignment() (*ConsumerGroupMemberAssignment, error) {
+ if len(gmd.MemberAssignment) == 0 {
+ return nil, nil
+ }
+ assignment := new(ConsumerGroupMemberAssignment)
+ err := decode(gmd.MemberAssignment, assignment, nil)
+ return assignment, err
+}
+
+func (gmd *GroupMemberDescription) GetMemberMetadata() (*ConsumerGroupMemberMetadata, error) {
+ if len(gmd.MemberMetadata) == 0 {
+ return nil, nil
+ }
+ metadata := new(ConsumerGroupMemberMetadata)
+ err := decode(gmd.MemberMetadata, metadata, nil)
+ return metadata, err
+}
diff --git a/vendor/github.com/IBM/sarama/describe_log_dirs_request.go b/vendor/github.com/IBM/sarama/describe_log_dirs_request.go
new file mode 100644
index 0000000..32354f6
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/describe_log_dirs_request.go
@@ -0,0 +1,124 @@
+package sarama
+
+// DescribeLogDirsRequest is a describe request to get partitions' log size
+type DescribeLogDirsRequest struct {
+ // Version 0 and 1 are equal
+ // The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
+ Version int16
+
+ // If this is an empty array, all topics will be queried
+ DescribeTopics []DescribeLogDirsRequestTopic
+}
+
+func (r *DescribeLogDirsRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+// DescribeLogDirsRequestTopic is a describe request about the log dir of one or more partitions within a Topic
+type DescribeLogDirsRequestTopic struct {
+ Topic string
+ PartitionIDs []int32
+}
+
+func (r *DescribeLogDirsRequest) encode(pe packetEncoder) error {
+ length := len(r.DescribeTopics)
+ if length == 0 {
+ // In order to query all topics we must send null
+ length = -1
+ }
+ if err := pe.putArrayLength(length); err != nil {
+ return err
+ }
+
+ for _, d := range r.DescribeTopics {
+ if err := pe.putString(d.Topic); err != nil {
+ return err
+ }
+
+ if err := pe.putInt32Array(d.PartitionIDs); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *DescribeLogDirsRequest) decode(pd packetDecoder, version int16) error {
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if n == -1 {
+ n = 0
+ }
+
+ topics := make([]DescribeLogDirsRequestTopic, n)
+ for i := 0; i < n; i++ {
+ topics[i] = DescribeLogDirsRequestTopic{}
+
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ topics[i].Topic = topic
+
+ pIDs, err := pd.getInt32Array()
+ if err != nil {
+ return err
+ }
+ topics[i].PartitionIDs = pIDs
+ _, err = pd.getEmptyTaggedFieldArray()
+ if err != nil {
+ return err
+ }
+ }
+ r.DescribeTopics = topics
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *DescribeLogDirsRequest) key() int16 {
+ return apiKeyDescribeLogDirs
+}
+
+func (r *DescribeLogDirsRequest) version() int16 {
+ return r.Version
+}
+
+func (r *DescribeLogDirsRequest) headerVersion() int16 {
+ if r.Version >= 2 {
+ return 2
+ }
+ return 1
+}
+
+func (r *DescribeLogDirsRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 4
+}
+
+func (r *DescribeLogDirsRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *DescribeLogDirsRequest) isFlexibleVersion(version int16) bool {
+ return version >= 2
+}
+
+func (r *DescribeLogDirsRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 4:
+ return V3_3_0_0
+ case 3:
+ return V3_2_0_0
+ case 2:
+ return V2_6_0_0
+ case 1:
+ return V2_0_0_0
+ default:
+ return V1_0_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/describe_log_dirs_response.go b/vendor/github.com/IBM/sarama/describe_log_dirs_response.go
new file mode 100644
index 0000000..ebae414
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/describe_log_dirs_response.go
@@ -0,0 +1,306 @@
+package sarama
+
+import "time"
+
+type DescribeLogDirsResponse struct {
+ ThrottleTime time.Duration
+
+ // Version 0 and 1 are equal
+ // The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
+ Version int16
+
+ LogDirs []DescribeLogDirsResponseDirMetadata
+
+ ErrorCode KError
+}
+
+func (r *DescribeLogDirsResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *DescribeLogDirsResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(r.ThrottleTime)
+
+ if r.Version >= 3 {
+ pe.putKError(r.ErrorCode)
+ }
+
+ if err := pe.putArrayLength(len(r.LogDirs)); err != nil {
+ return err
+ }
+
+ for _, dir := range r.LogDirs {
+ if err := dir.encode(pe, r.Version); err != nil {
+ return err
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *DescribeLogDirsResponse) decode(pd packetDecoder, version int16) (err error) {
+ if r.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ if version >= 3 {
+ r.ErrorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+ }
+
+ // Decode array of DescribeLogDirsResponseDirMetadata
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.LogDirs = make([]DescribeLogDirsResponseDirMetadata, n)
+ for i := 0; i < n; i++ {
+ dir := DescribeLogDirsResponseDirMetadata{}
+ if err := dir.decode(pd, version); err != nil {
+ return err
+ }
+ r.LogDirs[i] = dir
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *DescribeLogDirsResponse) key() int16 {
+ return apiKeyDescribeLogDirs
+}
+
+func (r *DescribeLogDirsResponse) version() int16 {
+ return r.Version
+}
+
+func (r *DescribeLogDirsResponse) headerVersion() int16 {
+ if r.Version >= 2 {
+ return 1
+ }
+ return 0
+}
+
+func (r *DescribeLogDirsResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 4
+}
+
+func (r *DescribeLogDirsResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *DescribeLogDirsResponse) isFlexibleVersion(version int16) bool {
+ return version >= 2
+}
+
+func (r *DescribeLogDirsResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 4:
+ return V3_3_0_0
+ case 3:
+ return V3_2_0_0
+ case 2:
+ return V2_6_0_0
+ case 1:
+ return V2_0_0_0
+ default:
+ return V1_0_0_0
+ }
+}
+
+func (r *DescribeLogDirsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
+
+type DescribeLogDirsResponseDirMetadata struct {
+ ErrorCode KError
+
+ // The absolute log directory path
+ Path string
+ Topics []DescribeLogDirsResponseTopic
+
+ TotalBytes int64
+ UsableBytes int64
+}
+
+func (r *DescribeLogDirsResponseDirMetadata) encode(pe packetEncoder, version int16) error {
+ pe.putKError(r.ErrorCode)
+
+ err := pe.putString(r.Path)
+ if err != nil {
+ return err
+ }
+
+ if err := pe.putArrayLength(len(r.Topics)); err != nil {
+ return err
+ }
+ for _, topic := range r.Topics {
+ if err := topic.encode(pe, version); err != nil {
+ return err
+ }
+ }
+
+ if version >= 4 {
+ pe.putInt64(r.TotalBytes)
+ pe.putInt64(r.UsableBytes)
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *DescribeLogDirsResponseDirMetadata) decode(pd packetDecoder, version int16) (err error) {
+ r.ErrorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ path, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ r.Path = path
+
+ // Decode array of DescribeLogDirsResponseTopic
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Topics = make([]DescribeLogDirsResponseTopic, n)
+ for i := 0; i < n; i++ {
+ t := DescribeLogDirsResponseTopic{}
+
+ if err := t.decode(pd, version); err != nil {
+ return err
+ }
+
+ r.Topics[i] = t
+ }
+
+ if version >= 4 {
+ totalBytes, err := pd.getInt64()
+ if err != nil {
+ return err
+ }
+ r.TotalBytes = totalBytes
+ usableBytes, err := pd.getInt64()
+ if err != nil {
+ return err
+ }
+ r.UsableBytes = usableBytes
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+// DescribeLogDirsResponseTopic contains a topic's partitions descriptions
+type DescribeLogDirsResponseTopic struct {
+ Topic string
+ Partitions []DescribeLogDirsResponsePartition
+}
+
+func (r *DescribeLogDirsResponseTopic) encode(pe packetEncoder, version int16) error {
+ if err := pe.putString(r.Topic); err != nil {
+ return err
+ }
+ if err := pe.putArrayLength(len(r.Partitions)); err != nil {
+ return err
+ }
+ for _, partition := range r.Partitions {
+ if err := partition.encode(pe, version); err != nil {
+ return err
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *DescribeLogDirsResponseTopic) decode(pd packetDecoder, version int16) error {
+ t, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ r.Topic = t
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Partitions = make([]DescribeLogDirsResponsePartition, n)
+ for i := 0; i < n; i++ {
+ p := DescribeLogDirsResponsePartition{}
+ if err := p.decode(pd, version); err != nil {
+ return err
+ }
+ r.Partitions[i] = p
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+// DescribeLogDirsResponsePartition describes a partition's log directory
+type DescribeLogDirsResponsePartition struct {
+ PartitionID int32
+
+ // The size of the log segments of the partition in bytes.
+ Size int64
+
+ // The lag of the log's LEO w.r.t. partition's HW (if it is the current log for the partition) or
+ // current replica's LEO (if it is the future log for the partition)
+ OffsetLag int64
+
+ // True if this log is created by AlterReplicaLogDirsRequest and will replace the current log of
+ // the replica in the future.
+ IsTemporary bool
+}
+
+func (r *DescribeLogDirsResponsePartition) encode(pe packetEncoder, version int16) error {
+ isFlexible := version >= 2
+ pe.putInt32(r.PartitionID)
+ pe.putInt64(r.Size)
+ pe.putInt64(r.OffsetLag)
+ pe.putBool(r.IsTemporary)
+ if isFlexible {
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ return nil
+}
+
+func (r *DescribeLogDirsResponsePartition) decode(pd packetDecoder, version int16) error {
+ pID, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ r.PartitionID = pID
+
+ size, err := pd.getInt64()
+ if err != nil {
+ return err
+ }
+ r.Size = size
+
+ lag, err := pd.getInt64()
+ if err != nil {
+ return err
+ }
+ r.OffsetLag = lag
+
+ isTemp, err := pd.getBool()
+ if err != nil {
+ return err
+ }
+ r.IsTemporary = isTemp
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
diff --git a/vendor/github.com/IBM/sarama/describe_user_scram_credentials_request.go b/vendor/github.com/IBM/sarama/describe_user_scram_credentials_request.go
new file mode 100644
index 0000000..4de3dcf
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/describe_user_scram_credentials_request.go
@@ -0,0 +1,86 @@
+package sarama
+
+// DescribeUserScramCredentialsRequest is a request to get list of SCRAM user names
+type DescribeUserScramCredentialsRequest struct {
+ // Version 0 is currently only supported
+ Version int16
+
+ // If this is an empty array, all users will be queried
+ DescribeUsers []DescribeUserScramCredentialsRequestUser
+}
+
+func (r *DescribeUserScramCredentialsRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+// DescribeUserScramCredentialsRequestUser is a describe request about specific user name
+type DescribeUserScramCredentialsRequestUser struct {
+ Name string
+}
+
+func (r *DescribeUserScramCredentialsRequest) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(r.DescribeUsers)); err != nil {
+ return err
+ }
+ for _, d := range r.DescribeUsers {
+ if err := pe.putString(d.Name); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *DescribeUserScramCredentialsRequest) decode(pd packetDecoder, version int16) error {
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if n == -1 {
+ n = 0
+ }
+
+ r.DescribeUsers = make([]DescribeUserScramCredentialsRequestUser, n)
+ for i := 0; i < n; i++ {
+ r.DescribeUsers[i] = DescribeUserScramCredentialsRequestUser{}
+ if r.DescribeUsers[i].Name, err = pd.getString(); err != nil {
+ return err
+ }
+ if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *DescribeUserScramCredentialsRequest) key() int16 {
+ return apiKeyDescribeUserScramCredentials
+}
+
+func (r *DescribeUserScramCredentialsRequest) version() int16 {
+ return r.Version
+}
+
+func (r *DescribeUserScramCredentialsRequest) headerVersion() int16 {
+ return 2
+}
+
+func (r *DescribeUserScramCredentialsRequest) isValidVersion() bool {
+ return r.Version == 0
+}
+
+func (r *DescribeUserScramCredentialsRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *DescribeUserScramCredentialsRequest) isFlexibleVersion(version int16) bool {
+ return version >= 0
+}
+
+func (r *DescribeUserScramCredentialsRequest) requiredVersion() KafkaVersion {
+ return V2_7_0_0
+}
diff --git a/vendor/github.com/IBM/sarama/describe_user_scram_credentials_response.go b/vendor/github.com/IBM/sarama/describe_user_scram_credentials_response.go
new file mode 100644
index 0000000..6852e0c
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/describe_user_scram_credentials_response.go
@@ -0,0 +1,186 @@
+package sarama
+
+import "time"
+
+type ScramMechanismType int8
+
+const (
+ SCRAM_MECHANISM_UNKNOWN ScramMechanismType = iota // 0
+ SCRAM_MECHANISM_SHA_256 // 1
+ SCRAM_MECHANISM_SHA_512 // 2
+)
+
+func (s ScramMechanismType) String() string {
+ switch s {
+ case 1:
+ return SASLTypeSCRAMSHA256
+ case 2:
+ return SASLTypeSCRAMSHA512
+ default:
+ return "Unknown"
+ }
+}
+
+type DescribeUserScramCredentialsResponse struct {
+ // Version 0 is currently only supported
+ Version int16
+
+ ThrottleTime time.Duration
+
+ ErrorCode KError
+ ErrorMessage *string
+
+ Results []*DescribeUserScramCredentialsResult
+}
+
+func (r *DescribeUserScramCredentialsResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+type DescribeUserScramCredentialsResult struct {
+ User string
+
+ ErrorCode KError
+ ErrorMessage *string
+
+ CredentialInfos []*UserScramCredentialsResponseInfo
+}
+
+type UserScramCredentialsResponseInfo struct {
+ Mechanism ScramMechanismType
+ Iterations int32
+}
+
+func (r *DescribeUserScramCredentialsResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(r.ThrottleTime)
+
+ pe.putKError(r.ErrorCode)
+ if err := pe.putNullableString(r.ErrorMessage); err != nil {
+ return err
+ }
+
+ if err := pe.putArrayLength(len(r.Results)); err != nil {
+ return err
+ }
+ for _, u := range r.Results {
+ if err := pe.putString(u.User); err != nil {
+ return err
+ }
+ pe.putInt16(int16(u.ErrorCode))
+ if err := pe.putNullableString(u.ErrorMessage); err != nil {
+ return err
+ }
+
+ if err := pe.putArrayLength(len(u.CredentialInfos)); err != nil {
+ return err
+ }
+ for _, c := range u.CredentialInfos {
+ pe.putInt8(int8(c.Mechanism))
+ pe.putInt32(c.Iterations)
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *DescribeUserScramCredentialsResponse) decode(pd packetDecoder, version int16) (err error) {
+ if r.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ r.ErrorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if r.ErrorMessage, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ numUsers, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if numUsers > 0 {
+ r.Results = make([]*DescribeUserScramCredentialsResult, numUsers)
+ for i := 0; i < numUsers; i++ {
+ r.Results[i] = &DescribeUserScramCredentialsResult{}
+ if r.Results[i].User, err = pd.getString(); err != nil {
+ return err
+ }
+
+ r.Results[i].ErrorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+ if r.Results[i].ErrorMessage, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ numCredentialInfos, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Results[i].CredentialInfos = make([]*UserScramCredentialsResponseInfo, numCredentialInfos)
+ for j := 0; j < numCredentialInfos; j++ {
+ r.Results[i].CredentialInfos[j] = &UserScramCredentialsResponseInfo{}
+ scramMechanism, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ r.Results[i].CredentialInfos[j].Mechanism = ScramMechanismType(scramMechanism)
+ if r.Results[i].CredentialInfos[j].Iterations, err = pd.getInt32(); err != nil {
+ return err
+ }
+ if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+
+ if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *DescribeUserScramCredentialsResponse) key() int16 {
+ return apiKeyDescribeUserScramCredentials
+}
+
+func (r *DescribeUserScramCredentialsResponse) version() int16 {
+ return r.Version
+}
+
+func (r *DescribeUserScramCredentialsResponse) headerVersion() int16 {
+ return 2
+}
+
+func (r *DescribeUserScramCredentialsResponse) isValidVersion() bool {
+ return r.Version == 0
+}
+
+func (r *DescribeUserScramCredentialsResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *DescribeUserScramCredentialsResponse) isFlexibleVersion(version int16) bool {
+ return version >= 0
+}
+
+func (r *DescribeUserScramCredentialsResponse) requiredVersion() KafkaVersion {
+ return V2_7_0_0
+}
+
+func (r *DescribeUserScramCredentialsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/dev.yml b/vendor/github.com/IBM/sarama/dev.yml
new file mode 100644
index 0000000..e2acb38
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/dev.yml
@@ -0,0 +1,10 @@
+name: sarama
+
+up:
+ - go:
+ version: '1.17.6'
+
+commands:
+ test:
+ run: make test
+ desc: 'run unit tests'
diff --git a/vendor/github.com/IBM/sarama/docker-compose.yml b/vendor/github.com/IBM/sarama/docker-compose.yml
new file mode 100644
index 0000000..3d23396
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/docker-compose.yml
@@ -0,0 +1,175 @@
+x-zookeeper-base: &zookeeper-base
+ image: 'docker.io/library/zookeeper:3.7.2'
+ init: true
+ restart: always
+ profiles:
+ - zookeeper
+ environment: &zookeeper-base-env
+ ZOO_SERVERS: 'server.1=zookeeper-1:2888:3888 server.2=zookeeper-2:2888:3888 server.3=zookeeper-3:2888:3888'
+ ZOO_CFG_EXTRA: 'clientPort=2181 peerPort=2888 leaderPort=3888'
+ ZOO_INIT_LIMIT: '10'
+ ZOO_SYNC_LIMIT: '5'
+ ZOO_MAX_CLIENT_CNXNS: '0'
+ ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
+
+x-kafka-base: &kafka-base
+ image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.9.1}'
+ init: true
+ build:
+ context: .
+ dockerfile: Dockerfile.kafka
+ args:
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.9.1}
+ SCALA_VERSION: ${SCALA_VERSION:-2.13}
+ depends_on:
+ - toxiproxy
+ restart: always
+ environment: &kafka-base-env
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.9.1}
+ KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
+ KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: '2'
+ KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '2'
+ KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '6000'
+ KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '6000'
+ KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
+ KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
+ KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
+ KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+ KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
+ KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
+ KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT,CONTROLLER:PLAINTEXT'
+ # ZooKeeper-specific
+ KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
+ # KRaft-specific
+ KAFKA_CFG_PROCESS_ROLES: 'broker,controller'
+ KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: '1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093,4@kafka-4:9093,5@kafka-5:9093'
+ KAFKA_CFG_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+ KAFKA_CFG_CLUSTER_ID: 'cDZEekk4T3hTNGVlNzB3LUtUbkxaQQo='
+
+services:
+ zookeeper-1:
+ <<: *zookeeper-base
+ container_name: 'zookeeper-1'
+ environment:
+ <<: *zookeeper-base-env
+ ZOO_MY_ID: '1'
+
+ zookeeper-2:
+ <<: *zookeeper-base
+ container_name: 'zookeeper-2'
+ environment:
+ <<: *zookeeper-base-env
+ ZOO_MY_ID: '2'
+
+ zookeeper-3:
+ <<: *zookeeper-base
+ container_name: 'zookeeper-3'
+ environment:
+ <<: *zookeeper-base-env
+ ZOO_MY_ID: '3'
+
+ kafka-1:
+ <<: *kafka-base
+ container_name: 'kafka-1'
+ healthcheck:
+ test: ['CMD', '/opt/kafka-${KAFKA_VERSION:-3.9.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-1:9091']
+ interval: 15s
+ timeout: 15s
+ retries: 10
+ start_period: 360s
+ environment:
+ <<: *kafka-base-env
+ KAFKA_CFG_BROKER_ID: '1'
+ KAFKA_CFG_NODE_ID: '1'
+ KAFKA_CFG_BROKER_RACK: '1'
+ KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091,CONTROLLER://:9093'
+ KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
+
+ kafka-2:
+ <<: *kafka-base
+ container_name: 'kafka-2'
+ healthcheck:
+ test: ['CMD', '/opt/kafka-${KAFKA_VERSION:-3.9.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-2:9091']
+ interval: 15s
+ timeout: 15s
+ retries: 10
+ start_period: 360s
+ environment:
+ <<: *kafka-base-env
+ KAFKA_CFG_BROKER_ID: '2'
+ KAFKA_CFG_NODE_ID: '2'
+ KAFKA_CFG_BROKER_RACK: '2'
+ KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092,CONTROLLER://:9093'
+ KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
+
+ kafka-3:
+ <<: *kafka-base
+ container_name: 'kafka-3'
+ healthcheck:
+ test: ['CMD', '/opt/kafka-${KAFKA_VERSION:-3.9.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-3:9091']
+ interval: 15s
+ timeout: 15s
+ retries: 10
+ start_period: 360s
+ environment:
+ <<: *kafka-base-env
+ KAFKA_CFG_BROKER_ID: '3'
+ KAFKA_CFG_NODE_ID: '3'
+ KAFKA_CFG_BROKER_RACK: '3'
+ KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093,CONTROLLER://:9093'
+ KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
+
+ kafka-4:
+ <<: *kafka-base
+ container_name: 'kafka-4'
+ healthcheck:
+ test: ['CMD', '/opt/kafka-${KAFKA_VERSION:-3.9.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-4:9091']
+ interval: 15s
+ timeout: 15s
+ retries: 10
+ start_period: 360s
+ environment:
+ <<: *kafka-base-env
+ KAFKA_CFG_BROKER_ID: '4'
+ KAFKA_CFG_NODE_ID: '4'
+ KAFKA_CFG_BROKER_RACK: '4'
+ KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094,CONTROLLER://:9093'
+ KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
+
+ kafka-5:
+ <<: *kafka-base
+ container_name: 'kafka-5'
+ healthcheck:
+ test: ['CMD', '/opt/kafka-${KAFKA_VERSION:-3.9.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-5:9091']
+ interval: 15s
+ timeout: 15s
+ retries: 10
+ start_period: 360s
+ environment:
+ <<: *kafka-base-env
+ KAFKA_CFG_BROKER_ID: '5'
+ KAFKA_CFG_NODE_ID: '5'
+ KAFKA_CFG_BROKER_RACK: '5'
+ KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095,CONTROLLER://:9093'
+ KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'
+
+ toxiproxy:
+ container_name: 'toxiproxy'
+ image: 'ghcr.io/shopify/toxiproxy:2.12.0'
+ init: true
+ healthcheck:
+ test: ['CMD', '/toxiproxy-cli', 'l']
+ interval: 15s
+ timeout: 15s
+ retries: 3
+ start_period: 30s
+ ports:
+ # The tests themselves actually start the proxies on these ports
+ - '29091:29091'
+ - '29092:29092'
+ - '29093:29093'
+ - '29094:29094'
+ - '29095:29095'
+
+ # This is the toxiproxy API port
+ - '8474:8474'
diff --git a/vendor/github.com/IBM/sarama/elect_leaders_request.go b/vendor/github.com/IBM/sarama/elect_leaders_request.go
new file mode 100644
index 0000000..01b9dd5
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/elect_leaders_request.go
@@ -0,0 +1,125 @@
+package sarama
+
+type ElectLeadersRequest struct {
+ Version int16
+ Type ElectionType
+ TopicPartitions map[string][]int32
+ TimeoutMs int32
+}
+
+func (r *ElectLeadersRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *ElectLeadersRequest) encode(pe packetEncoder) error {
+ if r.Version > 0 {
+ pe.putInt8(int8(r.Type))
+ }
+
+ if err := pe.putArrayLength(len(r.TopicPartitions)); err != nil {
+ return err
+ }
+
+ for topic, partitions := range r.TopicPartitions {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+
+ if err := pe.putInt32Array(partitions); err != nil {
+ return err
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putInt32(r.TimeoutMs)
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *ElectLeadersRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ if r.Version > 0 {
+ t, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ r.Type = ElectionType(t)
+ }
+
+ topicCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if topicCount > 0 {
+ r.TopicPartitions = make(map[string][]int32)
+ for i := 0; i < topicCount; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ partitionCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ partitions := make([]int32, partitionCount)
+ for j := 0; j < partitionCount; j++ {
+ partition, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ partitions[j] = partition
+ }
+ r.TopicPartitions[topic] = partitions
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+ }
+
+ r.TimeoutMs, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *ElectLeadersRequest) key() int16 {
+ return apiKeyElectLeaders
+}
+
+func (r *ElectLeadersRequest) version() int16 {
+ return r.Version
+}
+
+func (r *ElectLeadersRequest) headerVersion() int16 {
+ return 2
+}
+
+func (r *ElectLeadersRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 2
+}
+
+func (r *ElectLeadersRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *ElectLeadersRequest) isFlexibleVersion(version int16) bool {
+ return version >= 2
+}
+
+func (r *ElectLeadersRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 2:
+ return V2_4_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_10_0_0
+ default:
+ return V2_4_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/elect_leaders_response.go b/vendor/github.com/IBM/sarama/elect_leaders_response.go
new file mode 100644
index 0000000..90a3fab
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/elect_leaders_response.go
@@ -0,0 +1,161 @@
+package sarama
+
+import "time"
+
+type PartitionResult struct {
+ ErrorCode KError
+ ErrorMessage *string
+}
+
+func (b *PartitionResult) encode(pe packetEncoder, version int16) error {
+ pe.putKError(b.ErrorCode)
+ if err := pe.putNullableString(b.ErrorMessage); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (b *PartitionResult) decode(pd packetDecoder, version int16) (err error) {
+ b.ErrorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+ b.ErrorMessage, err = pd.getNullableString()
+ if err != nil {
+ return err
+ }
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+type ElectLeadersResponse struct {
+ Version int16
+ ThrottleTimeMs int32
+ ErrorCode KError
+ ReplicaElectionResults map[string]map[int32]*PartitionResult
+}
+
+func (r *ElectLeadersResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *ElectLeadersResponse) encode(pe packetEncoder) error {
+ pe.putInt32(r.ThrottleTimeMs)
+
+ if r.Version > 0 {
+ pe.putKError(r.ErrorCode)
+ }
+
+ if err := pe.putArrayLength(len(r.ReplicaElectionResults)); err != nil {
+ return err
+ }
+ for topic, partitions := range r.ReplicaElectionResults {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putArrayLength(len(partitions)); err != nil {
+ return err
+ }
+ for partition, result := range partitions {
+ pe.putInt32(partition)
+ if err := result.encode(pe, r.Version); err != nil {
+ return err
+ }
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *ElectLeadersResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ if r.ThrottleTimeMs, err = pd.getInt32(); err != nil {
+ return err
+ }
+ if r.Version > 0 {
+ r.ErrorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+ }
+
+ numTopics, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.ReplicaElectionResults = make(map[string]map[int32]*PartitionResult, numTopics)
+ for i := 0; i < numTopics; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ numPartitions, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ r.ReplicaElectionResults[topic] = make(map[int32]*PartitionResult, numPartitions)
+ for j := 0; j < numPartitions; j++ {
+ partition, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ result := new(PartitionResult)
+ if err := result.decode(pd, r.Version); err != nil {
+ return err
+ }
+ r.ReplicaElectionResults[topic][partition] = result
+ }
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *ElectLeadersResponse) key() int16 {
+ return apiKeyElectLeaders
+}
+
+func (r *ElectLeadersResponse) version() int16 {
+ return r.Version
+}
+
+func (r *ElectLeadersResponse) headerVersion() int16 {
+ return 1
+}
+
+func (r *ElectLeadersResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 2
+}
+
+func (r *ElectLeadersResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *ElectLeadersResponse) isFlexibleVersion(version int16) bool {
+ return version >= 2
+}
+
+func (r *ElectLeadersResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 2:
+ return V2_4_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_10_0_0
+ default:
+ return V2_4_0_0
+ }
+}
+
+func (r *ElectLeadersResponse) throttleTime() time.Duration {
+ return time.Duration(r.ThrottleTimeMs) * time.Millisecond
+}
diff --git a/vendor/github.com/IBM/sarama/election_type.go b/vendor/github.com/IBM/sarama/election_type.go
new file mode 100644
index 0000000..01f3b65
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/election_type.go
@@ -0,0 +1,10 @@
+package sarama
+
+type ElectionType int8
+
+const (
+ // PreferredElection constant type
+ PreferredElection ElectionType = 0
+ // UncleanElection constant type
+ UncleanElection ElectionType = 1
+)
diff --git a/vendor/github.com/IBM/sarama/encoder_decoder.go b/vendor/github.com/IBM/sarama/encoder_decoder.go
new file mode 100644
index 0000000..ef60224
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/encoder_decoder.go
@@ -0,0 +1,134 @@
+package sarama
+
+import (
+ "fmt"
+
+ "github.com/rcrowley/go-metrics"
+)
+
+// Encoder is the interface that wraps the basic Encode method.
+// Anything implementing Encoder can be turned into bytes using Kafka's encoding rules.
+type encoder interface {
+ encode(pe packetEncoder) error
+}
+
+type encoderWithHeader interface {
+ encoder
+ headerVersion() int16
+}
+
+// Encode takes an Encoder and turns it into bytes while potentially recording metrics.
+func encode(e encoder, metricRegistry metrics.Registry) ([]byte, error) {
+ if e == nil {
+ return nil, nil
+ }
+
+ var prepEnc prepEncoder
+ var realEnc realEncoder
+
+ err := e.encode(prepareFlexibleEncoder(&prepEnc, e))
+ if err != nil {
+ return nil, err
+ }
+
+ if prepEnc.length < 0 || prepEnc.length > int(MaxRequestSize) {
+ return nil, PacketEncodingError{fmt.Sprintf("invalid request size (%d)", prepEnc.length)}
+ }
+
+ realEnc.raw = make([]byte, prepEnc.length)
+ realEnc.registry = metricRegistry
+ err = e.encode(prepareFlexibleEncoder(&realEnc, e))
+ if err != nil {
+ return nil, err
+ }
+
+ return realEnc.raw, nil
+}
+
+// decoder is the interface that wraps the basic Decode method.
+// Anything implementing Decoder can be extracted from bytes using Kafka's encoding rules.
+type decoder interface {
+ decode(pd packetDecoder) error
+}
+
+type versionedDecoder interface {
+ decode(pd packetDecoder, version int16) error
+}
+
+type flexibleVersion interface {
+ isFlexibleVersion(version int16) bool
+ isFlexible() bool
+}
+
+// decode takes bytes and a decoder and fills the fields of the decoder from the bytes,
+// interpreted using Kafka's encoding rules.
+func decode(buf []byte, in decoder, metricRegistry metrics.Registry) error {
+ if buf == nil {
+ return nil
+ }
+ helper := realDecoder{
+ raw: buf,
+ registry: metricRegistry,
+ }
+ err := in.decode(&helper)
+ if err != nil {
+ return err
+ }
+
+ if helper.off != len(buf) {
+ return PacketDecodingError{fmt.Sprintf("invalid length: buf=%d decoded=%d %#v", len(buf), helper.off, in)}
+ }
+
+ return nil
+}
+
+func versionedDecode(buf []byte, in versionedDecoder, version int16, metricRegistry metrics.Registry) error {
+ if buf == nil {
+ return nil
+ }
+
+ helper := prepareFlexibleDecoder(&realDecoder{
+ raw: buf,
+ registry: metricRegistry,
+ }, in, version)
+ err := in.decode(helper, version)
+ if err != nil {
+ return err
+ }
+
+ if remaining := helper.remaining(); remaining != 0 {
+ return PacketDecodingError{
+ Info: fmt.Sprintf("invalid length len=%d remaining=%d", len(buf), remaining),
+ }
+ }
+
+ return nil
+}
+
+func prepareFlexibleDecoder(pd *realDecoder, in versionedDecoder, version int16) packetDecoder {
+ if flexibleDecoder, ok := in.(flexibleVersion); ok && flexibleDecoder.isFlexibleVersion(version) {
+ return &realFlexibleDecoder{pd}
+ }
+ return pd
+}
+
+func prepareFlexibleEncoder(pe packetEncoder, req encoder) packetEncoder {
+ if flexibleEncoder, ok := req.(flexibleVersion); ok && flexibleEncoder.isFlexible() {
+ switch e := pe.(type) {
+ case *prepEncoder:
+ return &prepFlexibleEncoder{e}
+ case *realEncoder:
+ return &realFlexibleEncoder{e}
+ default:
+ return pe
+ }
+ }
+ return pe
+}
+
+func downgradeFlexibleDecoder(pd packetDecoder) packetDecoder {
+ if f, ok := pd.(*realFlexibleDecoder); ok {
+ return f.realDecoder
+ }
+ return pd
+}
diff --git a/vendor/github.com/IBM/sarama/end_txn_request.go b/vendor/github.com/IBM/sarama/end_txn_request.go
new file mode 100644
index 0000000..1f0bf8c
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/end_txn_request.go
@@ -0,0 +1,70 @@
+package sarama
+
+type EndTxnRequest struct {
+ Version int16
+ TransactionalID string
+ ProducerID int64
+ ProducerEpoch int16
+ TransactionResult bool
+}
+
+func (a *EndTxnRequest) setVersion(v int16) {
+ a.Version = v
+}
+
+func (a *EndTxnRequest) encode(pe packetEncoder) error {
+ if err := pe.putString(a.TransactionalID); err != nil {
+ return err
+ }
+
+ pe.putInt64(a.ProducerID)
+
+ pe.putInt16(a.ProducerEpoch)
+
+ pe.putBool(a.TransactionResult)
+
+ return nil
+}
+
+func (a *EndTxnRequest) decode(pd packetDecoder, version int16) (err error) {
+ if a.TransactionalID, err = pd.getString(); err != nil {
+ return err
+ }
+ if a.ProducerID, err = pd.getInt64(); err != nil {
+ return err
+ }
+ if a.ProducerEpoch, err = pd.getInt16(); err != nil {
+ return err
+ }
+ if a.TransactionResult, err = pd.getBool(); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (a *EndTxnRequest) key() int16 {
+ return apiKeyEndTxn
+}
+
+func (a *EndTxnRequest) version() int16 {
+ return a.Version
+}
+
+func (r *EndTxnRequest) headerVersion() int16 {
+ return 1
+}
+
+func (a *EndTxnRequest) isValidVersion() bool {
+ return a.Version >= 0 && a.Version <= 2
+}
+
+func (a *EndTxnRequest) requiredVersion() KafkaVersion {
+ switch a.Version {
+ case 2:
+ return V2_7_0_0
+ case 1:
+ return V2_0_0_0
+ default:
+ return V0_11_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/end_txn_response.go b/vendor/github.com/IBM/sarama/end_txn_response.go
new file mode 100644
index 0000000..259f18b
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/end_txn_response.go
@@ -0,0 +1,65 @@
+package sarama
+
+import (
+ "time"
+)
+
+type EndTxnResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ Err KError
+}
+
+func (e *EndTxnResponse) setVersion(v int16) {
+ e.Version = v
+}
+
+func (e *EndTxnResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(e.ThrottleTime)
+ pe.putKError(e.Err)
+ return nil
+}
+
+func (e *EndTxnResponse) decode(pd packetDecoder, version int16) (err error) {
+ if e.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ e.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (e *EndTxnResponse) key() int16 {
+ return apiKeyEndTxn
+}
+
+func (e *EndTxnResponse) version() int16 {
+ return e.Version
+}
+
+func (r *EndTxnResponse) headerVersion() int16 {
+ return 0
+}
+
+func (e *EndTxnResponse) isValidVersion() bool {
+ return e.Version >= 0 && e.Version <= 2
+}
+
+func (e *EndTxnResponse) requiredVersion() KafkaVersion {
+ switch e.Version {
+ case 2:
+ return V2_7_0_0
+ case 1:
+ return V2_0_0_0
+ default:
+ return V0_11_0_0
+ }
+}
+
+func (r *EndTxnResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/entrypoint.sh b/vendor/github.com/IBM/sarama/entrypoint.sh
new file mode 100644
index 0000000..9815d90
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/entrypoint.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+KAFKA_VERSION="${KAFKA_VERSION:-3.9.1}"
+KAFKA_HOME="/opt/kafka-${KAFKA_VERSION}"
+
+if [ ! -d "${KAFKA_HOME}" ]; then
+ echo 'Error: KAFKA_VERSION '$KAFKA_VERSION' not available in this image at '$KAFKA_HOME
+ exit 1
+fi
+
+cd "${KAFKA_HOME}" || exit 1
+
+# discard all empty/commented lines from default config and copy to /tmp
+sed -e '/^#/d' -e '/^$/d' config/server.properties >/tmp/server.properties
+
+echo "########################################################################" >>/tmp/server.properties
+
+# emulate kafka_configure_from_environment_variables from bitnami/bitnami-docker-kafka
+for var in "${!KAFKA_CFG_@}"; do
+ key="$(echo "$var" | sed -e 's/^KAFKA_CFG_//g' -e 's/_/\./g' -e 's/.*/\L&/')"
+ sed -e '/^'$key'/d' -i"" /tmp/server.properties
+ value="${!var}"
+ echo "$key=$value" >>/tmp/server.properties
+done
+
+# use KRaft if KAFKA_VERSION is 4.0.0 or newer
+if printf "%s\n4.0.0" "$KAFKA_VERSION" | sort -V | head -n1 | grep -q "^4.0.0$"; then
+ # remove zookeeper-only options from server.properties and setup storage
+ sed -e '/zookeeper.connect/d' \
+ -i /tmp/server.properties
+ bin/kafka-storage.sh format -t "$KAFKA_CFG_CLUSTER_ID" -c /tmp/server.properties --ignore-formatted
+else
+ # remove KRaft-only options from server.properties
+ sed -e '/process.roles/d' \
+ -e '/controller.quorum.voters/d' \
+ -e '/controller.listener.names/d' \
+ -e '/cluster.id/d' \
+ -i /tmp/server.properties
+fi
+
+sort /tmp/server.properties
+
+exec bin/kafka-server-start.sh /tmp/server.properties
diff --git a/vendor/github.com/IBM/sarama/errors.go b/vendor/github.com/IBM/sarama/errors.go
new file mode 100644
index 0000000..55af653
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/errors.go
@@ -0,0 +1,450 @@
+package sarama
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+)
+
+// ErrOutOfBrokers is the error returned when the client has run out of brokers to talk to because all of them errored
+// or otherwise failed to respond.
+var ErrOutOfBrokers = errors.New("kafka: client has run out of available brokers to talk to")
+
+// ErrBrokerNotFound is the error returned when there's no broker found for the requested ID.
+var ErrBrokerNotFound = errors.New("kafka: broker for ID is not found")
+
+// ErrClosedClient is the error returned when a method is called on a client that has been closed.
+var ErrClosedClient = errors.New("kafka: tried to use a client that was closed")
+
+// ErrIncompleteResponse is the error returned when the server returns a syntactically valid response, but it does
+// not contain the expected information.
+var ErrIncompleteResponse = errors.New("kafka: response did not contain all the expected topic/partition blocks")
+
+// ErrInvalidPartition is the error returned when a partitioner returns an invalid partition index
+// (meaning one outside of the range [0...numPartitions-1]).
+var ErrInvalidPartition = errors.New("kafka: partitioner returned an invalid partition index")
+
+// ErrAlreadyConnected is the error returned when calling Open() on a Broker that is already connected or connecting.
+var ErrAlreadyConnected = errors.New("kafka: broker connection already initiated")
+
+// ErrNotConnected is the error returned when trying to send or call Close() on a Broker that is not connected.
+var ErrNotConnected = errors.New("kafka: broker not connected")
+
+// ErrInsufficientData is returned when decoding and the packet is truncated. This can be expected
+// when requesting messages, since as an optimization the server is allowed to return a partial message at the end
+// of the message set.
+var ErrInsufficientData = errors.New("kafka: insufficient data to decode packet, more bytes expected")
+
+// ErrShuttingDown is returned when a producer receives a message during shutdown.
+var ErrShuttingDown = errors.New("kafka: message received by producer in process of shutting down")
+
+// ErrMessageTooLarge is returned when the next message to consume is larger than the configured Consumer.Fetch.Max
+var ErrMessageTooLarge = errors.New("kafka: message is larger than Consumer.Fetch.Max")
+
+// ErrConsumerOffsetNotAdvanced is returned when a partition consumer didn't advance its offset after parsing
+// a RecordBatch.
+var ErrConsumerOffsetNotAdvanced = errors.New("kafka: consumer offset was not advanced after a RecordBatch")
+
+// ErrControllerNotAvailable is returned when server didn't give correct controller id. May be kafka server's version
+// is lower than 0.10.0.0.
+var ErrControllerNotAvailable = errors.New("kafka: controller is not available")
+
+// ErrNoTopicsToUpdateMetadata is returned when Meta.Full is set to false but no specific topics were found to update
+// the metadata.
+var ErrNoTopicsToUpdateMetadata = errors.New("kafka: no specific topics to update metadata")
+
+// ErrUnknownScramMechanism is returned when user tries to AlterUserScramCredentials with unknown SCRAM mechanism
+var ErrUnknownScramMechanism = errors.New("kafka: unknown SCRAM mechanism provided")
+
+// ErrReassignPartitions is returned when altering partition assignments for a topic fails
+var ErrReassignPartitions = errors.New("failed to reassign partitions for topic")
+
+// ErrDeleteRecords is the type of error returned when fail to delete the required records
+var ErrDeleteRecords = errors.New("kafka server: failed to delete records")
+
+// ErrCreateACLs is the type of error returned when ACL creation failed
+var ErrCreateACLs = errors.New("kafka server: failed to create one or more ACL rules")
+
+// ErrAddPartitionsToTxn is returned when AddPartitionsToTxn failed multiple times
+var ErrAddPartitionsToTxn = errors.New("transaction manager: failed to send partitions to transaction")
+
+// ErrTxnOffsetCommit is returned when TxnOffsetCommit failed multiple times
+var ErrTxnOffsetCommit = errors.New("transaction manager: failed to send offsets to transaction")
+
+// ErrTransactionNotReady when transaction status is invalid for the current action.
+var ErrTransactionNotReady = errors.New("transaction manager: transaction is not ready")
+
+// ErrNonTransactedProducer when calling BeginTxn, CommitTxn or AbortTxn on a non transactional producer.
+var ErrNonTransactedProducer = errors.New("transaction manager: you need to add TransactionalID to producer")
+
+// ErrTransitionNotAllowed when txnmgr state transition is not valid.
+var ErrTransitionNotAllowed = errors.New("transaction manager: invalid transition attempted")
+
+// ErrCannotTransitionNilError when transition is attempted with an nil error.
+var ErrCannotTransitionNilError = errors.New("transaction manager: cannot transition with a nil error")
+
+// ErrTxnUnableToParseResponse when response is nil
+var ErrTxnUnableToParseResponse = errors.New("transaction manager: unable to parse response")
+
+// ErrUnknownMessage when the protocol message key is not recognized
+var ErrUnknownMessage = errors.New("kafka: unknown protocol message key")
+
+// MultiErrorFormat specifies the formatter applied to format multierrors.
+//
+// Deprecated: Please use [errors.Join] instead.
+func MultiErrorFormat(es []error) string {
+ if len(es) == 1 {
+ return es[0].Error()
+ }
+
+ points := make([]string, len(es))
+ for i, err := range es {
+ points[i] = fmt.Sprintf("* %s", err)
+ }
+
+ return fmt.Sprintf(
+ "%d errors occurred:\n\t%s\n",
+ len(es), strings.Join(points, "\n\t"))
+}
+
+type sentinelError struct {
+ sentinel error
+ wrapped error
+}
+
+func (err sentinelError) Error() string {
+ if err.wrapped != nil {
+ return fmt.Sprintf("%s: %v", err.sentinel, err.wrapped)
+ } else {
+ return fmt.Sprintf("%s", err.sentinel)
+ }
+}
+
+func (err sentinelError) Is(target error) bool {
+ return errors.Is(err.sentinel, target) || errors.Is(err.wrapped, target)
+}
+
+func (err sentinelError) Unwrap() error {
+ return err.wrapped
+}
+
+func Wrap(sentinel error, wrapped ...error) sentinelError {
+ return sentinelError{sentinel: sentinel, wrapped: errors.Join(wrapped...)}
+}
+
+// PacketEncodingError is returned from a failure while encoding a Kafka packet. This can happen, for example,
+// if you try to encode a string over 2^15 characters in length, since Kafka's encoding rules do not permit that.
+type PacketEncodingError struct {
+ Info string
+}
+
+func (err PacketEncodingError) Error() string {
+ return fmt.Sprintf("kafka: error encoding packet: %s", err.Info)
+}
+
+// PacketDecodingError is returned when there was an error (other than truncated data) decoding the Kafka broker's response.
+// This can be a bad CRC or length field, or any other invalid value.
+type PacketDecodingError struct {
+ Info string
+}
+
+func (err PacketDecodingError) Error() string {
+ return fmt.Sprintf("kafka: error decoding packet: %s", err.Info)
+}
+
+// ConfigurationError is the type of error returned from a constructor (e.g. NewClient, or NewConsumer)
+// when the specified configuration is invalid.
+type ConfigurationError string
+
+func (err ConfigurationError) Error() string {
+ return "kafka: invalid configuration (" + string(err) + ")"
+}
+
+// KError is the type of error that can be returned directly by the Kafka broker.
+// See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes
+type KError int16
+
+// Numeric error codes returned by the Kafka server.
+const (
+ ErrUnknown KError = -1 // Errors.UNKNOWN_SERVER_ERROR
+ ErrNoError KError = 0 // Errors.NONE
+ ErrOffsetOutOfRange KError = 1 // Errors.OFFSET_OUT_OF_RANGE
+ ErrInvalidMessage KError = 2 // Errors.CORRUPT_MESSAGE
+ ErrUnknownTopicOrPartition KError = 3 // Errors.UNKNOWN_TOPIC_OR_PARTITION
+ ErrInvalidMessageSize KError = 4 // Errors.INVALID_FETCH_SIZE
+ ErrLeaderNotAvailable KError = 5 // Errors.LEADER_NOT_AVAILABLE
+ ErrNotLeaderForPartition KError = 6 // Errors.NOT_LEADER_OR_FOLLOWER
+ ErrRequestTimedOut KError = 7 // Errors.REQUEST_TIMED_OUT
+ ErrBrokerNotAvailable KError = 8 // Errors.BROKER_NOT_AVAILABLE
+ ErrReplicaNotAvailable KError = 9 // Errors.REPLICA_NOT_AVAILABLE
+ ErrMessageSizeTooLarge KError = 10 // Errors.MESSAGE_TOO_LARGE
+ ErrStaleControllerEpochCode KError = 11 // Errors.STALE_CONTROLLER_EPOCH
+ ErrOffsetMetadataTooLarge KError = 12 // Errors.OFFSET_METADATA_TOO_LARGE
+ ErrNetworkException KError = 13 // Errors.NETWORK_EXCEPTION
+ ErrOffsetsLoadInProgress KError = 14 // Errors.COORDINATOR_LOAD_IN_PROGRESS
+ ErrConsumerCoordinatorNotAvailable KError = 15 // Errors.COORDINATOR_NOT_AVAILABLE
+ ErrNotCoordinatorForConsumer KError = 16 // Errors.NOT_COORDINATOR
+ ErrInvalidTopic KError = 17 // Errors.INVALID_TOPIC_EXCEPTION
+ ErrMessageSetSizeTooLarge KError = 18 // Errors.RECORD_LIST_TOO_LARGE
+ ErrNotEnoughReplicas KError = 19 // Errors.NOT_ENOUGH_REPLICAS
+ ErrNotEnoughReplicasAfterAppend KError = 20 // Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND
+ ErrInvalidRequiredAcks KError = 21 // Errors.INVALID_REQUIRED_ACKS
+ ErrIllegalGeneration KError = 22 // Errors.ILLEGAL_GENERATION
+ ErrInconsistentGroupProtocol KError = 23 // Errors.INCONSISTENT_GROUP_PROTOCOL
+ ErrInvalidGroupId KError = 24 // Errors.INVALID_GROUP_ID
+ ErrUnknownMemberId KError = 25 // Errors.UNKNOWN_MEMBER_ID
+ ErrInvalidSessionTimeout KError = 26 // Errors.INVALID_SESSION_TIMEOUT
+ ErrRebalanceInProgress KError = 27 // Errors.REBALANCE_IN_PROGRESS
+ ErrInvalidCommitOffsetSize KError = 28 // Errors.INVALID_COMMIT_OFFSET_SIZE
+ ErrTopicAuthorizationFailed KError = 29 // Errors.TOPIC_AUTHORIZATION_FAILED
+ ErrGroupAuthorizationFailed KError = 30 // Errors.GROUP_AUTHORIZATION_FAILED
+ ErrClusterAuthorizationFailed KError = 31 // Errors.CLUSTER_AUTHORIZATION_FAILED
+ ErrInvalidTimestamp KError = 32 // Errors.INVALID_TIMESTAMP
+ ErrUnsupportedSASLMechanism KError = 33 // Errors.UNSUPPORTED_SASL_MECHANISM
+ ErrIllegalSASLState KError = 34 // Errors.ILLEGAL_SASL_STATE
+ ErrUnsupportedVersion KError = 35 // Errors.UNSUPPORTED_VERSION
+ ErrTopicAlreadyExists KError = 36 // Errors.TOPIC_ALREADY_EXISTS
+ ErrInvalidPartitions KError = 37 // Errors.INVALID_PARTITIONS
+ ErrInvalidReplicationFactor KError = 38 // Errors.INVALID_REPLICATION_FACTOR
+ ErrInvalidReplicaAssignment KError = 39 // Errors.INVALID_REPLICA_ASSIGNMENT
+ ErrInvalidConfig KError = 40 // Errors.INVALID_CONFIG
+ ErrNotController KError = 41 // Errors.NOT_CONTROLLER
+ ErrInvalidRequest KError = 42 // Errors.INVALID_REQUEST
+ ErrUnsupportedForMessageFormat KError = 43 // Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT
+ ErrPolicyViolation KError = 44 // Errors.POLICY_VIOLATION
+ ErrOutOfOrderSequenceNumber KError = 45 // Errors.OUT_OF_ORDER_SEQUENCE_NUMBER
+ ErrDuplicateSequenceNumber KError = 46 // Errors.DUPLICATE_SEQUENCE_NUMBER
+ ErrInvalidProducerEpoch KError = 47 // Errors.INVALID_PRODUCER_EPOCH
+ ErrInvalidTxnState KError = 48 // Errors.INVALID_TXN_STATE
+ ErrInvalidProducerIDMapping KError = 49 // Errors.INVALID_PRODUCER_ID_MAPPING
+ ErrInvalidTransactionTimeout KError = 50 // Errors.INVALID_TRANSACTION_TIMEOUT
+ ErrConcurrentTransactions KError = 51 // Errors.CONCURRENT_TRANSACTIONS
+ ErrTransactionCoordinatorFenced KError = 52 // Errors.TRANSACTION_COORDINATOR_FENCED
+ ErrTransactionalIDAuthorizationFailed KError = 53 // Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED
+ ErrSecurityDisabled KError = 54 // Errors.SECURITY_DISABLED
+ ErrOperationNotAttempted KError = 55 // Errors.OPERATION_NOT_ATTEMPTED
+ ErrKafkaStorageError KError = 56 // Errors.KAFKA_STORAGE_ERROR
+ ErrLogDirNotFound KError = 57 // Errors.LOG_DIR_NOT_FOUND
+ ErrSASLAuthenticationFailed KError = 58 // Errors.SASL_AUTHENTICATION_FAILED
+ ErrUnknownProducerID KError = 59 // Errors.UNKNOWN_PRODUCER_ID
+ ErrReassignmentInProgress KError = 60 // Errors.REASSIGNMENT_IN_PROGRESS
+ ErrDelegationTokenAuthDisabled KError = 61 // Errors.DELEGATION_TOKEN_AUTH_DISABLED
+ ErrDelegationTokenNotFound KError = 62 // Errors.DELEGATION_TOKEN_NOT_FOUND
+ ErrDelegationTokenOwnerMismatch KError = 63 // Errors.DELEGATION_TOKEN_OWNER_MISMATCH
+ ErrDelegationTokenRequestNotAllowed KError = 64 // Errors.DELEGATION_TOKEN_REQUEST_NOT_ALLOWED
+ ErrDelegationTokenAuthorizationFailed KError = 65 // Errors.DELEGATION_TOKEN_AUTHORIZATION_FAILED
+ ErrDelegationTokenExpired KError = 66 // Errors.DELEGATION_TOKEN_EXPIRED
+ ErrInvalidPrincipalType KError = 67 // Errors.INVALID_PRINCIPAL_TYPE
+ ErrNonEmptyGroup KError = 68 // Errors.NON_EMPTY_GROUP
+ ErrGroupIDNotFound KError = 69 // Errors.GROUP_ID_NOT_FOUND
+ ErrFetchSessionIDNotFound KError = 70 // Errors.FETCH_SESSION_ID_NOT_FOUND
+ ErrInvalidFetchSessionEpoch KError = 71 // Errors.INVALID_FETCH_SESSION_EPOCH
+ ErrListenerNotFound KError = 72 // Errors.LISTENER_NOT_FOUND
+ ErrTopicDeletionDisabled KError = 73 // Errors.TOPIC_DELETION_DISABLED
+ ErrFencedLeaderEpoch KError = 74 // Errors.FENCED_LEADER_EPOCH
+ ErrUnknownLeaderEpoch KError = 75 // Errors.UNKNOWN_LEADER_EPOCH
+ ErrUnsupportedCompressionType KError = 76 // Errors.UNSUPPORTED_COMPRESSION_TYPE
+ ErrStaleBrokerEpoch KError = 77 // Errors.STALE_BROKER_EPOCH
+ ErrOffsetNotAvailable KError = 78 // Errors.OFFSET_NOT_AVAILABLE
+ ErrMemberIdRequired KError = 79 // Errors.MEMBER_ID_REQUIRED
+ ErrPreferredLeaderNotAvailable KError = 80 // Errors.PREFERRED_LEADER_NOT_AVAILABLE
+ ErrGroupMaxSizeReached KError = 81 // Errors.GROUP_MAX_SIZE_REACHED
+ ErrFencedInstancedId KError = 82 // Errors.FENCED_INSTANCE_ID
+ ErrEligibleLeadersNotAvailable KError = 83 // Errors.ELIGIBLE_LEADERS_NOT_AVAILABLE
+ ErrElectionNotNeeded KError = 84 // Errors.ELECTION_NOT_NEEDED
+ ErrNoReassignmentInProgress KError = 85 // Errors.NO_REASSIGNMENT_IN_PROGRESS
+ ErrGroupSubscribedToTopic KError = 86 // Errors.GROUP_SUBSCRIBED_TO_TOPIC
+ ErrInvalidRecord KError = 87 // Errors.INVALID_RECORD
+ ErrUnstableOffsetCommit KError = 88 // Errors.UNSTABLE_OFFSET_COMMIT
+ ErrThrottlingQuotaExceeded KError = 89 // Errors.THROTTLING_QUOTA_EXCEEDED
+ ErrProducerFenced KError = 90 // Errors.PRODUCER_FENCED
+)
+
+func (err KError) Error() string {
+ // Error messages stolen/adapted from
+ // https://kafka.apache.org/protocol#protocol_error_codes
+ switch err {
+ case ErrNoError:
+ return "kafka server: Not an error, why are you printing me?"
+ case ErrUnknown:
+ return "kafka server: Unexpected (unknown?) server error"
+ case ErrOffsetOutOfRange:
+ return "kafka server: The requested offset is outside the range of offsets maintained by the server for the given topic/partition"
+ case ErrInvalidMessage:
+ return "kafka server: Message contents does not match its CRC"
+ case ErrUnknownTopicOrPartition:
+ return "kafka server: Request was for a topic or partition that does not exist on this broker"
+ case ErrInvalidMessageSize:
+ return "kafka server: The message has a negative size"
+ case ErrLeaderNotAvailable:
+ return "kafka server: In the middle of a leadership election, there is currently no leader for this partition and hence it is unavailable for writes"
+ case ErrNotLeaderForPartition:
+ return "kafka server: Tried to send a message to a replica that is not the leader for some partition. Your metadata is out of date"
+ case ErrRequestTimedOut:
+ return "kafka server: Request exceeded the user-specified time limit in the request"
+ case ErrBrokerNotAvailable:
+ return "kafka server: Broker not available. Not a client facing error, we should never receive this!!!"
+ case ErrReplicaNotAvailable:
+ return "kafka server: Replica information not available, one or more brokers are down"
+ case ErrMessageSizeTooLarge:
+ return "kafka server: Message was too large, server rejected it to avoid allocation error"
+ case ErrStaleControllerEpochCode:
+ return "kafka server: StaleControllerEpochCode (internal error code for broker-to-broker communication)"
+ case ErrOffsetMetadataTooLarge:
+ return "kafka server: Specified a string larger than the configured maximum for offset metadata"
+ case ErrNetworkException:
+ return "kafka server: The server disconnected before a response was received"
+ case ErrOffsetsLoadInProgress:
+ return "kafka server: The coordinator is still loading offsets and cannot currently process requests"
+ case ErrConsumerCoordinatorNotAvailable:
+ return "kafka server: The coordinator is not available"
+ case ErrNotCoordinatorForConsumer:
+ return "kafka server: Request was for a consumer group that is not coordinated by this broker"
+ case ErrInvalidTopic:
+ return "kafka server: The request attempted to perform an operation on an invalid topic"
+ case ErrMessageSetSizeTooLarge:
+ return "kafka server: The request included message batch larger than the configured segment size on the server"
+ case ErrNotEnoughReplicas:
+ return "kafka server: Messages are rejected since there are fewer in-sync replicas than required"
+ case ErrNotEnoughReplicasAfterAppend:
+ return "kafka server: Messages are written to the log, but to fewer in-sync replicas than required"
+ case ErrInvalidRequiredAcks:
+ return "kafka server: The number of required acks is invalid (should be either -1, 0, or 1)"
+ case ErrIllegalGeneration:
+ return "kafka server: The provided generation id is not the current generation"
+ case ErrInconsistentGroupProtocol:
+ return "kafka server: The provider group protocol type is incompatible with the other members"
+ case ErrInvalidGroupId:
+ return "kafka server: The provided group id was empty"
+ case ErrUnknownMemberId:
+ return "kafka server: The provided member is not known in the current generation"
+ case ErrInvalidSessionTimeout:
+ return "kafka server: The provided session timeout is outside the allowed range"
+ case ErrRebalanceInProgress:
+ return "kafka server: A rebalance for the group is in progress. Please re-join the group"
+ case ErrInvalidCommitOffsetSize:
+ return "kafka server: The provided commit metadata was too large"
+ case ErrTopicAuthorizationFailed:
+ return "kafka server: The client is not authorized to access this topic"
+ case ErrGroupAuthorizationFailed:
+ return "kafka server: The client is not authorized to access this group"
+ case ErrClusterAuthorizationFailed:
+ return "kafka server: The client is not authorized to send this request type"
+ case ErrInvalidTimestamp:
+ return "kafka server: The timestamp of the message is out of acceptable range"
+ case ErrUnsupportedSASLMechanism:
+ return "kafka server: The broker does not support the requested SASL mechanism"
+ case ErrIllegalSASLState:
+ return "kafka server: Request is not valid given the current SASL state"
+ case ErrUnsupportedVersion:
+ return "kafka server: The version of API is not supported"
+ case ErrTopicAlreadyExists:
+ return "kafka server: Topic with this name already exists"
+ case ErrInvalidPartitions:
+ return "kafka server: Number of partitions is invalid"
+ case ErrInvalidReplicationFactor:
+ return "kafka server: Replication-factor is invalid"
+ case ErrInvalidReplicaAssignment:
+ return "kafka server: Replica assignment is invalid"
+ case ErrInvalidConfig:
+ return "kafka server: Configuration is invalid"
+ case ErrNotController:
+ return "kafka server: This is not the correct controller for this cluster"
+ case ErrInvalidRequest:
+ return "kafka server: This most likely occurs because of a request being malformed by the client library or the message was sent to an incompatible broker. See the broker logs for more details"
+ case ErrUnsupportedForMessageFormat:
+ return "kafka server: The requested operation is not supported by the message format version"
+ case ErrPolicyViolation:
+ return "kafka server: Request parameters do not satisfy the configured policy"
+ case ErrOutOfOrderSequenceNumber:
+ return "kafka server: The broker received an out of order sequence number"
+ case ErrDuplicateSequenceNumber:
+ return "kafka server: The broker received a duplicate sequence number"
+ case ErrInvalidProducerEpoch:
+ return "kafka server: Producer attempted an operation with an old epoch"
+ case ErrInvalidTxnState:
+ return "kafka server: The producer attempted a transactional operation in an invalid state"
+ case ErrInvalidProducerIDMapping:
+ return "kafka server: The producer attempted to use a producer id which is not currently assigned to its transactional id"
+ case ErrInvalidTransactionTimeout:
+ return "kafka server: The transaction timeout is larger than the maximum value allowed by the broker (as configured by max.transaction.timeout.ms)"
+ case ErrConcurrentTransactions:
+ return "kafka server: The producer attempted to update a transaction while another concurrent operation on the same transaction was ongoing"
+ case ErrTransactionCoordinatorFenced:
+ return "kafka server: The transaction coordinator sending a WriteTxnMarker is no longer the current coordinator for a given producer"
+ case ErrTransactionalIDAuthorizationFailed:
+ return "kafka server: Transactional ID authorization failed"
+ case ErrSecurityDisabled:
+ return "kafka server: Security features are disabled"
+ case ErrOperationNotAttempted:
+ return "kafka server: The broker did not attempt to execute this operation"
+ case ErrKafkaStorageError:
+ return "kafka server: Disk error when trying to access log file on the disk"
+ case ErrLogDirNotFound:
+ return "kafka server: The specified log directory is not found in the broker config"
+ case ErrSASLAuthenticationFailed:
+ return "kafka server: SASL Authentication failed"
+ case ErrUnknownProducerID:
+ return "kafka server: The broker could not locate the producer metadata associated with the Producer ID"
+ case ErrReassignmentInProgress:
+ return "kafka server: A partition reassignment is in progress"
+ case ErrDelegationTokenAuthDisabled:
+ return "kafka server: Delegation Token feature is not enabled"
+ case ErrDelegationTokenNotFound:
+ return "kafka server: Delegation Token is not found on server"
+ case ErrDelegationTokenOwnerMismatch:
+ return "kafka server: Specified Principal is not valid Owner/Renewer"
+ case ErrDelegationTokenRequestNotAllowed:
+ return "kafka server: Delegation Token requests are not allowed on PLAINTEXT/1-way SSL channels and on delegation token authenticated channels"
+ case ErrDelegationTokenAuthorizationFailed:
+ return "kafka server: Delegation Token authorization failed"
+ case ErrDelegationTokenExpired:
+ return "kafka server: Delegation Token is expired"
+ case ErrInvalidPrincipalType:
+ return "kafka server: Supplied principalType is not supported"
+ case ErrNonEmptyGroup:
+ return "kafka server: The group is not empty"
+ case ErrGroupIDNotFound:
+ return "kafka server: The group id does not exist"
+ case ErrFetchSessionIDNotFound:
+ return "kafka server: The fetch session ID was not found"
+ case ErrInvalidFetchSessionEpoch:
+ return "kafka server: The fetch session epoch is invalid"
+ case ErrListenerNotFound:
+ return "kafka server: There is no listener on the leader broker that matches the listener on which metadata request was processed"
+ case ErrTopicDeletionDisabled:
+ return "kafka server: Topic deletion is disabled"
+ case ErrFencedLeaderEpoch:
+ return "kafka server: The leader epoch in the request is older than the epoch on the broker"
+ case ErrUnknownLeaderEpoch:
+ return "kafka server: The leader epoch in the request is newer than the epoch on the broker"
+ case ErrUnsupportedCompressionType:
+ return "kafka server: The requesting client does not support the compression type of given partition"
+ case ErrStaleBrokerEpoch:
+ return "kafka server: Broker epoch has changed"
+ case ErrOffsetNotAvailable:
+ return "kafka server: The leader high watermark has not caught up from a recent leader election so the offsets cannot be guaranteed to be monotonically increasing"
+ case ErrMemberIdRequired:
+ return "kafka server: The group member needs to have a valid member id before actually entering a consumer group"
+ case ErrPreferredLeaderNotAvailable:
+ return "kafka server: The preferred leader was not available"
+ case ErrGroupMaxSizeReached:
+ return "kafka server: Consumer group The consumer group has reached its max size. already has the configured maximum number of members"
+ case ErrFencedInstancedId:
+ return "kafka server: The broker rejected this static consumer since another consumer with the same group.instance.id has registered with a different member.id"
+ case ErrEligibleLeadersNotAvailable:
+ return "kafka server: Eligible topic partition leaders are not available"
+ case ErrElectionNotNeeded:
+ return "kafka server: Leader election not needed for topic partition"
+ case ErrNoReassignmentInProgress:
+ return "kafka server: No partition reassignment is in progress"
+ case ErrGroupSubscribedToTopic:
+ return "kafka server: Deleting offsets of a topic is forbidden while the consumer group is actively subscribed to it"
+ case ErrInvalidRecord:
+ return "kafka server: This record has failed the validation on broker and hence will be rejected"
+ case ErrUnstableOffsetCommit:
+ return "kafka server: There are unstable offsets that need to be cleared"
+ }
+
+ return fmt.Sprintf("Unknown error, how did this happen? Error code = %d", err)
+}
diff --git a/vendor/github.com/IBM/sarama/fetch_request.go b/vendor/github.com/IBM/sarama/fetch_request.go
new file mode 100644
index 0000000..6d072dd
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/fetch_request.go
@@ -0,0 +1,340 @@
+package sarama
+
+import "fmt"
+
+type fetchRequestBlock struct {
+ Version int16
+ // currentLeaderEpoch contains the current leader epoch of the partition.
+ currentLeaderEpoch int32
+ // fetchOffset contains the message offset.
+ fetchOffset int64
+ // logStartOffset contains the earliest available offset of the follower
+ // replica. The field is only used when the request is sent by the
+ // follower.
+ logStartOffset int64
+ // maxBytes contains the maximum bytes to fetch from this partition. See
+ // KIP-74 for cases where this limit may not be honored.
+ maxBytes int32
+}
+
+func (b *fetchRequestBlock) encode(pe packetEncoder, version int16) error {
+ b.Version = version
+ if b.Version >= 9 {
+ pe.putInt32(b.currentLeaderEpoch)
+ }
+ pe.putInt64(b.fetchOffset)
+ if b.Version >= 5 {
+ pe.putInt64(b.logStartOffset)
+ }
+ pe.putInt32(b.maxBytes)
+ return nil
+}
+
+func (b *fetchRequestBlock) decode(pd packetDecoder, version int16) (err error) {
+ b.Version = version
+ if b.Version >= 9 {
+ if b.currentLeaderEpoch, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+ if b.fetchOffset, err = pd.getInt64(); err != nil {
+ return err
+ }
+ if b.Version >= 5 {
+ if b.logStartOffset, err = pd.getInt64(); err != nil {
+ return err
+ }
+ }
+ if b.maxBytes, err = pd.getInt32(); err != nil {
+ return err
+ }
+ return nil
+}
+
+// FetchRequest (API key 1) will fetch Kafka messages. Version 3 introduced the MaxBytes field. See
+// https://issues.apache.org/jira/browse/KAFKA-2063 for a discussion of the issues leading up to that. The KIP is at
+// https://cwiki.apache.org/confluence/display/KAFKA/KIP-74%3A+Add+Fetch+Response+Size+Limit+in+Bytes
+type FetchRequest struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // ReplicaID contains the broker ID of the follower, of -1 if this request
+ // is from a consumer.
+ // ReplicaID int32
+ // MaxWaitTime contains the maximum time in milliseconds to wait for the response.
+ MaxWaitTime int32
+ // MinBytes contains the minimum bytes to accumulate in the response.
+ MinBytes int32
+ // MaxBytes contains the maximum bytes to fetch. See KIP-74 for cases
+ // where this limit may not be honored.
+ MaxBytes int32
+ // Isolation contains a This setting controls the visibility of
+ // transactional records. Using READ_UNCOMMITTED (isolation_level = 0)
+ // makes all records visible. With READ_COMMITTED (isolation_level = 1),
+ // non-transactional and COMMITTED transactional records are visible. To be
+ // more concrete, READ_COMMITTED returns all data from offsets smaller than
+ // the current LSO (last stable offset), and enables the inclusion of the
+ // list of aborted transactions in the result, which allows consumers to
+ // discard ABORTED transactional records
+ Isolation IsolationLevel
+ // SessionID contains the fetch session ID.
+ SessionID int32
+ // SessionEpoch contains the epoch of the partition leader as known to the
+ // follower replica or a consumer.
+ SessionEpoch int32
+ // blocks contains the topics to fetch.
+ blocks map[string]map[int32]*fetchRequestBlock
+ // forgotten contains in an incremental fetch request, the partitions to remove.
+ forgotten map[string][]int32
+ // RackID contains a Rack ID of the consumer making this request
+ RackID string
+}
+
+func (r *FetchRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+type IsolationLevel int8
+
+const (
+ ReadUncommitted IsolationLevel = iota
+ ReadCommitted
+)
+
+func (r *FetchRequest) encode(pe packetEncoder) (err error) {
+ metricRegistry := pe.metricRegistry()
+
+ pe.putInt32(-1) // ReplicaID is always -1 for clients
+ pe.putInt32(r.MaxWaitTime)
+ pe.putInt32(r.MinBytes)
+ if r.Version >= 3 {
+ pe.putInt32(r.MaxBytes)
+ }
+ if r.Version >= 4 {
+ pe.putInt8(int8(r.Isolation))
+ }
+ if r.Version >= 7 {
+ pe.putInt32(r.SessionID)
+ pe.putInt32(r.SessionEpoch)
+ }
+ err = pe.putArrayLength(len(r.blocks))
+ if err != nil {
+ return err
+ }
+ for topic, blocks := range r.blocks {
+ err = pe.putString(topic)
+ if err != nil {
+ return err
+ }
+ err = pe.putArrayLength(len(blocks))
+ if err != nil {
+ return err
+ }
+ for partition, block := range blocks {
+ pe.putInt32(partition)
+ err = block.encode(pe, r.Version)
+ if err != nil {
+ return err
+ }
+ }
+ getOrRegisterTopicMeter("consumer-fetch-rate", topic, metricRegistry).Mark(1)
+ }
+ if r.Version >= 7 {
+ err = pe.putArrayLength(len(r.forgotten))
+ if err != nil {
+ return err
+ }
+ for topic, partitions := range r.forgotten {
+ err = pe.putString(topic)
+ if err != nil {
+ return err
+ }
+ err = pe.putArrayLength(len(partitions))
+ if err != nil {
+ return err
+ }
+ for _, partition := range partitions {
+ pe.putInt32(partition)
+ }
+ }
+ }
+ if r.Version >= 11 {
+ err = pe.putString(r.RackID)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (r *FetchRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+
+ if _, err = pd.getInt32(); err != nil {
+ return err
+ }
+ if r.MaxWaitTime, err = pd.getInt32(); err != nil {
+ return err
+ }
+ if r.MinBytes, err = pd.getInt32(); err != nil {
+ return err
+ }
+ if r.Version >= 3 {
+ if r.MaxBytes, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+ if r.Version >= 4 {
+ isolation, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ r.Isolation = IsolationLevel(isolation)
+ }
+ if r.Version >= 7 {
+ r.SessionID, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+ r.SessionEpoch, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+ }
+ topicCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if topicCount == 0 {
+ return nil
+ }
+ r.blocks = make(map[string]map[int32]*fetchRequestBlock)
+ for i := 0; i < topicCount; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ partitionCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ r.blocks[topic] = make(map[int32]*fetchRequestBlock)
+ for j := 0; j < partitionCount; j++ {
+ partition, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ fetchBlock := &fetchRequestBlock{}
+ if err = fetchBlock.decode(pd, r.Version); err != nil {
+ return err
+ }
+ r.blocks[topic][partition] = fetchBlock
+ }
+ }
+
+ if r.Version >= 7 {
+ forgottenCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ r.forgotten = make(map[string][]int32)
+ for i := 0; i < forgottenCount; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ partitionCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if partitionCount < 0 {
+ return fmt.Errorf("partitionCount %d is invalid", partitionCount)
+ }
+ r.forgotten[topic] = make([]int32, partitionCount)
+
+ for j := 0; j < partitionCount; j++ {
+ partition, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ r.forgotten[topic][j] = partition
+ }
+ }
+ }
+
+ if r.Version >= 11 {
+ r.RackID, err = pd.getString()
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (r *FetchRequest) key() int16 {
+ return apiKeyFetch
+}
+
+func (r *FetchRequest) version() int16 {
+ return r.Version
+}
+
+func (r *FetchRequest) headerVersion() int16 {
+ return 1
+}
+
+func (r *FetchRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 11
+}
+
+func (r *FetchRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 11:
+ return V2_3_0_0
+ case 9, 10:
+ return V2_1_0_0
+ case 8:
+ return V2_0_0_0
+ case 7:
+ return V1_1_0_0
+ case 6:
+ return V1_0_0_0
+ case 4, 5:
+ return V0_11_0_0
+ case 3:
+ return V0_10_1_0
+ case 2:
+ return V0_10_0_0
+ case 1:
+ return V0_9_0_0
+ case 0:
+ return V0_8_2_0
+ default:
+ return V2_3_0_0
+ }
+}
+
+func (r *FetchRequest) AddBlock(topic string, partitionID int32, fetchOffset int64, maxBytes int32, leaderEpoch int32) {
+ if r.blocks == nil {
+ r.blocks = make(map[string]map[int32]*fetchRequestBlock)
+ }
+
+ if r.Version >= 7 && r.forgotten == nil {
+ r.forgotten = make(map[string][]int32)
+ }
+
+ if r.blocks[topic] == nil {
+ r.blocks[topic] = make(map[int32]*fetchRequestBlock)
+ }
+
+ tmp := new(fetchRequestBlock)
+ tmp.Version = r.Version
+ tmp.maxBytes = maxBytes
+ tmp.fetchOffset = fetchOffset
+ if r.Version >= 9 {
+ tmp.currentLeaderEpoch = leaderEpoch
+ }
+
+ r.blocks[topic][partitionID] = tmp
+}
diff --git a/vendor/github.com/IBM/sarama/fetch_response.go b/vendor/github.com/IBM/sarama/fetch_response.go
new file mode 100644
index 0000000..95de90e
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/fetch_response.go
@@ -0,0 +1,605 @@
+package sarama
+
+import (
+ "errors"
+ "sort"
+ "time"
+
+ "github.com/rcrowley/go-metrics"
+)
+
+const (
+ invalidLeaderEpoch = -1
+ invalidPreferredReplicaID = -1
+)
+
+type AbortedTransaction struct {
+ // ProducerID contains the producer id associated with the aborted transaction.
+ ProducerID int64
+ // FirstOffset contains the first offset in the aborted transaction.
+ FirstOffset int64
+}
+
+func (t *AbortedTransaction) decode(pd packetDecoder) (err error) {
+ if t.ProducerID, err = pd.getInt64(); err != nil {
+ return err
+ }
+
+ if t.FirstOffset, err = pd.getInt64(); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (t *AbortedTransaction) encode(pe packetEncoder) (err error) {
+ pe.putInt64(t.ProducerID)
+ pe.putInt64(t.FirstOffset)
+
+ return nil
+}
+
+type FetchResponseBlock struct {
+ // Err contains the error code, or 0 if there was no fetch error.
+ Err KError
+ // HighWatermarkOffset contains the current high water mark.
+ HighWaterMarkOffset int64
+ // LastStableOffset contains the last stable offset (or LSO) of the
+ // partition. This is the last offset such that the state of all
+ // transactional records prior to this offset have been decided (ABORTED or
+ // COMMITTED)
+ LastStableOffset int64
+ // LogStartOffset contains the current log start offset.
+ LogStartOffset int64
+ // AbortedTransactions contains the aborted transactions.
+ AbortedTransactions []*AbortedTransaction
+ // PreferredReadReplica contains the preferred read replica for the
+ // consumer to use on its next fetch request
+ PreferredReadReplica int32
+ // RecordsSet contains the record data.
+ RecordsSet []*Records
+
+ Partial bool
+ Records *Records // deprecated: use FetchResponseBlock.RecordsSet
+
+ // recordsNextOffset contains the next consecutive offset following this response block.
+ // This field is computed locally and is not part of the server's binary response.
+ recordsNextOffset *int64
+}
+
+func (b *FetchResponseBlock) decode(pd packetDecoder, version int16) (err error) {
+ metricRegistry := pd.metricRegistry()
+ var sizeMetric metrics.Histogram
+ if metricRegistry != nil {
+ sizeMetric = getOrRegisterHistogram("consumer-fetch-response-size", metricRegistry)
+ }
+
+ b.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ b.HighWaterMarkOffset, err = pd.getInt64()
+ if err != nil {
+ return err
+ }
+
+ if version >= 4 {
+ b.LastStableOffset, err = pd.getInt64()
+ if err != nil {
+ return err
+ }
+
+ if version >= 5 {
+ b.LogStartOffset, err = pd.getInt64()
+ if err != nil {
+ return err
+ }
+ }
+
+ numTransact, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if numTransact >= 0 {
+ b.AbortedTransactions = make([]*AbortedTransaction, numTransact)
+ }
+
+ for i := 0; i < numTransact; i++ {
+ transact := new(AbortedTransaction)
+ if err = transact.decode(pd); err != nil {
+ return err
+ }
+ b.AbortedTransactions[i] = transact
+ }
+ }
+
+ if version >= 11 {
+ b.PreferredReadReplica, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+ } else {
+ b.PreferredReadReplica = -1
+ }
+
+ recordsSize, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ if sizeMetric != nil {
+ sizeMetric.Update(int64(recordsSize))
+ }
+
+ recordsDecoder, err := pd.getSubset(int(recordsSize))
+ if err != nil {
+ return err
+ }
+
+ b.RecordsSet = []*Records{}
+
+ for recordsDecoder.remaining() > 0 {
+ records := &Records{}
+ if err := records.decode(recordsDecoder); err != nil {
+ // If we have at least one decoded records, this is not an error
+ if errors.Is(err, ErrInsufficientData) {
+ if len(b.RecordsSet) == 0 {
+ b.Partial = true
+ }
+ break
+ }
+ return err
+ }
+
+ b.recordsNextOffset, err = records.nextOffset()
+ if err != nil {
+ return err
+ }
+
+ partial, err := records.isPartial()
+ if err != nil {
+ return err
+ }
+
+ n, err := records.numRecords()
+ if err != nil {
+ return err
+ }
+
+ if n > 0 || (partial && len(b.RecordsSet) == 0) {
+ b.RecordsSet = append(b.RecordsSet, records)
+
+ if b.Records == nil {
+ b.Records = records
+ }
+ }
+
+ overflow, err := records.isOverflow()
+ if err != nil {
+ return err
+ }
+
+ if partial || overflow {
+ break
+ }
+ }
+
+ return nil
+}
+
+func (b *FetchResponseBlock) numRecords() (int, error) {
+ sum := 0
+
+ for _, records := range b.RecordsSet {
+ count, err := records.numRecords()
+ if err != nil {
+ return 0, err
+ }
+
+ sum += count
+ }
+
+ return sum, nil
+}
+
+func (b *FetchResponseBlock) isPartial() (bool, error) {
+ if b.Partial {
+ return true, nil
+ }
+
+ if len(b.RecordsSet) == 1 {
+ return b.RecordsSet[0].isPartial()
+ }
+
+ return false, nil
+}
+
+func (b *FetchResponseBlock) encode(pe packetEncoder, version int16) (err error) {
+ pe.putKError(b.Err)
+
+ pe.putInt64(b.HighWaterMarkOffset)
+
+ if version >= 4 {
+ pe.putInt64(b.LastStableOffset)
+
+ if version >= 5 {
+ pe.putInt64(b.LogStartOffset)
+ }
+
+ if err = pe.putArrayLength(len(b.AbortedTransactions)); err != nil {
+ return err
+ }
+ for _, transact := range b.AbortedTransactions {
+ if err = transact.encode(pe); err != nil {
+ return err
+ }
+ }
+ }
+
+ if version >= 11 {
+ pe.putInt32(b.PreferredReadReplica)
+ }
+
+ pe.push(&lengthField{})
+ for _, records := range b.RecordsSet {
+ err = records.encode(pe)
+ if err != nil {
+ return err
+ }
+ }
+ return pe.pop()
+}
+
+func (b *FetchResponseBlock) getAbortedTransactions() []*AbortedTransaction {
+ // I can't find any doc that guarantee the field `fetchResponse.AbortedTransactions` is ordered
+ // plus Java implementation use a PriorityQueue based on `FirstOffset`. I guess we have to order it ourself
+ at := b.AbortedTransactions
+ sort.Slice(
+ at,
+ func(i, j int) bool { return at[i].FirstOffset < at[j].FirstOffset },
+ )
+ return at
+}
+
+type FetchResponse struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // ThrottleTime contains the duration in milliseconds for which the request
+ // was throttled due to a quota violation, or zero if the request did not
+ // violate any quota.
+ ThrottleTime time.Duration
+ // ErrorCode contains the top level response error code.
+ ErrorCode int16
+ // SessionID contains the fetch session ID, or 0 if this is not part of a fetch session.
+ SessionID int32
+ // Blocks contains the response topics.
+ Blocks map[string]map[int32]*FetchResponseBlock
+
+ LogAppendTime bool
+ Timestamp time.Time
+}
+
+func (r *FetchResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *FetchResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+
+ if r.Version >= 1 {
+ if r.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+ }
+
+ if r.Version >= 7 {
+ r.ErrorCode, err = pd.getInt16()
+ if err != nil {
+ return err
+ }
+ r.SessionID, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+ }
+
+ numTopics, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Blocks = make(map[string]map[int32]*FetchResponseBlock, numTopics)
+ for i := 0; i < numTopics; i++ {
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ numBlocks, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Blocks[name] = make(map[int32]*FetchResponseBlock, numBlocks)
+
+ for j := 0; j < numBlocks; j++ {
+ id, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ block := new(FetchResponseBlock)
+ err = block.decode(pd, version)
+ if err != nil {
+ return err
+ }
+ r.Blocks[name][id] = block
+ }
+ }
+
+ return nil
+}
+
+func (r *FetchResponse) encode(pe packetEncoder) (err error) {
+ if r.Version >= 1 {
+ pe.putDurationMs(r.ThrottleTime)
+ }
+
+ if r.Version >= 7 {
+ pe.putInt16(r.ErrorCode)
+ pe.putInt32(r.SessionID)
+ }
+
+ err = pe.putArrayLength(len(r.Blocks))
+ if err != nil {
+ return err
+ }
+
+ for topic, partitions := range r.Blocks {
+ err = pe.putString(topic)
+ if err != nil {
+ return err
+ }
+
+ err = pe.putArrayLength(len(partitions))
+ if err != nil {
+ return err
+ }
+
+ for id, block := range partitions {
+ pe.putInt32(id)
+ err = block.encode(pe, r.Version)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func (r *FetchResponse) key() int16 {
+ return apiKeyFetch
+}
+
+func (r *FetchResponse) version() int16 {
+ return r.Version
+}
+
+func (r *FetchResponse) headerVersion() int16 {
+ return 0
+}
+
+func (r *FetchResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 11
+}
+
+func (r *FetchResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 11:
+ return V2_3_0_0
+ case 9, 10:
+ return V2_1_0_0
+ case 8:
+ return V2_0_0_0
+ case 7:
+ return V1_1_0_0
+ case 6:
+ return V1_0_0_0
+ case 4, 5:
+ return V0_11_0_0
+ case 3:
+ return V0_10_1_0
+ case 2:
+ return V0_10_0_0
+ case 1:
+ return V0_9_0_0
+ case 0:
+ return V0_8_2_0
+ default:
+ return V2_3_0_0
+ }
+}
+
+func (r *FetchResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
+
+func (r *FetchResponse) GetBlock(topic string, partition int32) *FetchResponseBlock {
+ if r.Blocks == nil {
+ return nil
+ }
+
+ if r.Blocks[topic] == nil {
+ return nil
+ }
+
+ return r.Blocks[topic][partition]
+}
+
+func (r *FetchResponse) AddError(topic string, partition int32, err KError) {
+ if r.Blocks == nil {
+ r.Blocks = make(map[string]map[int32]*FetchResponseBlock)
+ }
+ partitions, ok := r.Blocks[topic]
+ if !ok {
+ partitions = make(map[int32]*FetchResponseBlock)
+ r.Blocks[topic] = partitions
+ }
+ frb, ok := partitions[partition]
+ if !ok {
+ frb = new(FetchResponseBlock)
+ partitions[partition] = frb
+ }
+ frb.Err = err
+}
+
+func (r *FetchResponse) getOrCreateBlock(topic string, partition int32) *FetchResponseBlock {
+ if r.Blocks == nil {
+ r.Blocks = make(map[string]map[int32]*FetchResponseBlock)
+ }
+ partitions, ok := r.Blocks[topic]
+ if !ok {
+ partitions = make(map[int32]*FetchResponseBlock)
+ r.Blocks[topic] = partitions
+ }
+ frb, ok := partitions[partition]
+ if !ok {
+ frb = new(FetchResponseBlock)
+ partitions[partition] = frb
+ }
+
+ return frb
+}
+
+func encodeKV(key, value Encoder) ([]byte, []byte) {
+ var kb []byte
+ var vb []byte
+ if key != nil {
+ kb, _ = key.Encode()
+ }
+ if value != nil {
+ vb, _ = value.Encode()
+ }
+
+ return kb, vb
+}
+
+func (r *FetchResponse) AddMessageWithTimestamp(topic string, partition int32, key, value Encoder, offset int64, timestamp time.Time, version int8) {
+ frb := r.getOrCreateBlock(topic, partition)
+ kb, vb := encodeKV(key, value)
+ if r.LogAppendTime {
+ timestamp = r.Timestamp
+ }
+ msg := &Message{Key: kb, Value: vb, LogAppendTime: r.LogAppendTime, Timestamp: timestamp, Version: version}
+ msgBlock := &MessageBlock{Msg: msg, Offset: offset}
+ if len(frb.RecordsSet) == 0 {
+ records := newLegacyRecords(&MessageSet{})
+ frb.RecordsSet = []*Records{&records}
+ }
+ set := frb.RecordsSet[0].MsgSet
+ set.Messages = append(set.Messages, msgBlock)
+}
+
+func (r *FetchResponse) AddRecordWithTimestamp(topic string, partition int32, key, value Encoder, offset int64, timestamp time.Time) {
+ frb := r.getOrCreateBlock(topic, partition)
+ kb, vb := encodeKV(key, value)
+ if len(frb.RecordsSet) == 0 {
+ records := newDefaultRecords(&RecordBatch{Version: 2, LogAppendTime: r.LogAppendTime, FirstTimestamp: timestamp, MaxTimestamp: r.Timestamp})
+ frb.RecordsSet = []*Records{&records}
+ }
+ batch := frb.RecordsSet[0].RecordBatch
+ rec := &Record{Key: kb, Value: vb, OffsetDelta: offset, TimestampDelta: timestamp.Sub(batch.FirstTimestamp)}
+ batch.addRecord(rec)
+}
+
+// AddRecordBatchWithTimestamp is similar to AddRecordWithTimestamp
+// But instead of appending 1 record to a batch, it append a new batch containing 1 record to the fetchResponse
+// Since transaction are handled on batch level (the whole batch is either committed or aborted), use this to test transactions
+func (r *FetchResponse) AddRecordBatchWithTimestamp(topic string, partition int32, key, value Encoder, offset int64, producerID int64, isTransactional bool, timestamp time.Time) {
+ frb := r.getOrCreateBlock(topic, partition)
+ kb, vb := encodeKV(key, value)
+
+ records := newDefaultRecords(&RecordBatch{Version: 2, LogAppendTime: r.LogAppendTime, FirstTimestamp: timestamp, MaxTimestamp: r.Timestamp})
+ batch := &RecordBatch{
+ Version: 2,
+ LogAppendTime: r.LogAppendTime,
+ FirstTimestamp: timestamp,
+ MaxTimestamp: r.Timestamp,
+ FirstOffset: offset,
+ LastOffsetDelta: 0,
+ ProducerID: producerID,
+ IsTransactional: isTransactional,
+ }
+ rec := &Record{Key: kb, Value: vb, OffsetDelta: 0, TimestampDelta: timestamp.Sub(batch.FirstTimestamp)}
+ batch.addRecord(rec)
+ records.RecordBatch = batch
+
+ frb.RecordsSet = append(frb.RecordsSet, &records)
+}
+
+func (r *FetchResponse) AddControlRecordWithTimestamp(topic string, partition int32, offset int64, producerID int64, recordType ControlRecordType, timestamp time.Time) {
+ frb := r.getOrCreateBlock(topic, partition)
+
+ // batch
+ batch := &RecordBatch{
+ Version: 2,
+ LogAppendTime: r.LogAppendTime,
+ FirstTimestamp: timestamp,
+ MaxTimestamp: r.Timestamp,
+ FirstOffset: offset,
+ LastOffsetDelta: 0,
+ ProducerID: producerID,
+ IsTransactional: true,
+ Control: true,
+ }
+
+ // records
+ records := newDefaultRecords(nil)
+ records.RecordBatch = batch
+
+ // record
+ crAbort := ControlRecord{
+ Version: 0,
+ Type: recordType,
+ }
+ crKey := &realEncoder{raw: make([]byte, 4)}
+ crValue := &realEncoder{raw: make([]byte, 6)}
+ crAbort.encode(crKey, crValue)
+ rec := &Record{Key: ByteEncoder(crKey.raw), Value: ByteEncoder(crValue.raw), OffsetDelta: 0, TimestampDelta: timestamp.Sub(batch.FirstTimestamp)}
+ batch.addRecord(rec)
+
+ frb.RecordsSet = append(frb.RecordsSet, &records)
+}
+
+func (r *FetchResponse) AddMessage(topic string, partition int32, key, value Encoder, offset int64) {
+ r.AddMessageWithTimestamp(topic, partition, key, value, offset, time.Time{}, 0)
+}
+
+func (r *FetchResponse) AddRecord(topic string, partition int32, key, value Encoder, offset int64) {
+ r.AddRecordWithTimestamp(topic, partition, key, value, offset, time.Time{})
+}
+
+func (r *FetchResponse) AddRecordBatch(topic string, partition int32, key, value Encoder, offset int64, producerID int64, isTransactional bool) {
+ r.AddRecordBatchWithTimestamp(topic, partition, key, value, offset, producerID, isTransactional, time.Time{})
+}
+
+func (r *FetchResponse) AddControlRecord(topic string, partition int32, offset int64, producerID int64, recordType ControlRecordType) {
+ // define controlRecord key and value
+ r.AddControlRecordWithTimestamp(topic, partition, offset, producerID, recordType, time.Time{})
+}
+
+func (r *FetchResponse) SetLastOffsetDelta(topic string, partition int32, offset int32) {
+ frb := r.getOrCreateBlock(topic, partition)
+ if len(frb.RecordsSet) == 0 {
+ records := newDefaultRecords(&RecordBatch{Version: 2})
+ frb.RecordsSet = []*Records{&records}
+ }
+ batch := frb.RecordsSet[0].RecordBatch
+ batch.LastOffsetDelta = offset
+}
+
+func (r *FetchResponse) SetLastStableOffset(topic string, partition int32, offset int64) {
+ frb := r.getOrCreateBlock(topic, partition)
+ frb.LastStableOffset = offset
+}
diff --git a/vendor/github.com/IBM/sarama/find_coordinator_request.go b/vendor/github.com/IBM/sarama/find_coordinator_request.go
new file mode 100644
index 0000000..04ffc6f
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/find_coordinator_request.go
@@ -0,0 +1,75 @@
+package sarama
+
+type CoordinatorType int8
+
+const (
+ CoordinatorGroup CoordinatorType = iota
+ CoordinatorTransaction
+)
+
+type FindCoordinatorRequest struct {
+ Version int16
+ CoordinatorKey string
+ CoordinatorType CoordinatorType
+}
+
+func (f *FindCoordinatorRequest) setVersion(v int16) {
+ f.Version = v
+}
+
+func (f *FindCoordinatorRequest) encode(pe packetEncoder) error {
+ if err := pe.putString(f.CoordinatorKey); err != nil {
+ return err
+ }
+
+ if f.Version >= 1 {
+ pe.putInt8(int8(f.CoordinatorType))
+ }
+
+ return nil
+}
+
+func (f *FindCoordinatorRequest) decode(pd packetDecoder, version int16) (err error) {
+ if f.CoordinatorKey, err = pd.getString(); err != nil {
+ return err
+ }
+
+ if version >= 1 {
+ f.Version = version
+ coordinatorType, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+
+ f.CoordinatorType = CoordinatorType(coordinatorType)
+ }
+
+ return nil
+}
+
+func (f *FindCoordinatorRequest) key() int16 {
+ return apiKeyFindCoordinator
+}
+
+func (f *FindCoordinatorRequest) version() int16 {
+ return f.Version
+}
+
+func (r *FindCoordinatorRequest) headerVersion() int16 {
+ return 1
+}
+
+func (f *FindCoordinatorRequest) isValidVersion() bool {
+ return f.Version >= 0 && f.Version <= 2
+}
+
+func (f *FindCoordinatorRequest) requiredVersion() KafkaVersion {
+ switch f.Version {
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ default:
+ return V0_8_2_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/find_coordinator_response.go b/vendor/github.com/IBM/sarama/find_coordinator_response.go
new file mode 100644
index 0000000..1fb8a28
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/find_coordinator_response.go
@@ -0,0 +1,107 @@
+package sarama
+
+import (
+ "time"
+)
+
+var NoNode = &Broker{id: -1, addr: ":-1"}
+
+type FindCoordinatorResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ Err KError
+ ErrMsg *string
+ Coordinator *Broker
+}
+
+func (f *FindCoordinatorResponse) setVersion(v int16) {
+ f.Version = v
+}
+
+func (f *FindCoordinatorResponse) decode(pd packetDecoder, version int16) (err error) {
+ if version >= 1 {
+ f.Version = version
+
+ if f.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+ }
+
+ f.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if version >= 1 {
+ if f.ErrMsg, err = pd.getNullableString(); err != nil {
+ return err
+ }
+ }
+
+ coordinator := new(Broker)
+ // The version is hardcoded to 0, as version 1 of the Broker-decode
+ // contains the rack-field which is not present in the FindCoordinatorResponse.
+ if err := coordinator.decode(pd, 0); err != nil {
+ return err
+ }
+ if coordinator.addr == ":0" {
+ return nil
+ }
+ f.Coordinator = coordinator
+
+ return nil
+}
+
+func (f *FindCoordinatorResponse) encode(pe packetEncoder) error {
+ if f.Version >= 1 {
+ pe.putDurationMs(f.ThrottleTime)
+ }
+
+ pe.putKError(f.Err)
+
+ if f.Version >= 1 {
+ if err := pe.putNullableString(f.ErrMsg); err != nil {
+ return err
+ }
+ }
+
+ coordinator := f.Coordinator
+ if coordinator == nil {
+ coordinator = NoNode
+ }
+ if err := coordinator.encode(pe, 0); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (f *FindCoordinatorResponse) key() int16 {
+ return apiKeyFindCoordinator
+}
+
+func (f *FindCoordinatorResponse) version() int16 {
+ return f.Version
+}
+
+func (r *FindCoordinatorResponse) headerVersion() int16 {
+ return 0
+}
+
+func (f *FindCoordinatorResponse) isValidVersion() bool {
+ return f.Version >= 0 && f.Version <= 2
+}
+
+func (f *FindCoordinatorResponse) requiredVersion() KafkaVersion {
+ switch f.Version {
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ default:
+ return V0_8_2_0
+ }
+}
+
+func (r *FindCoordinatorResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/gssapi_kerberos.go b/vendor/github.com/IBM/sarama/gssapi_kerberos.go
new file mode 100644
index 0000000..93ea921
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/gssapi_kerberos.go
@@ -0,0 +1,321 @@
+package sarama
+
+import (
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "net"
+ "strings"
+ "time"
+
+ "github.com/jcmturner/gofork/encoding/asn1"
+ "github.com/jcmturner/gokrb5/v8/asn1tools"
+ "github.com/jcmturner/gokrb5/v8/gssapi"
+ "github.com/jcmturner/gokrb5/v8/iana/chksumtype"
+ "github.com/jcmturner/gokrb5/v8/iana/keyusage"
+ "github.com/jcmturner/gokrb5/v8/messages"
+ "github.com/jcmturner/gokrb5/v8/types"
+)
+
+const (
+ TOK_ID_KRB_AP_REQ = 256
+ GSS_API_GENERIC_TAG = 0x60
+ KRB5_USER_AUTH = 1
+ KRB5_KEYTAB_AUTH = 2
+ KRB5_CCACHE_AUTH = 3
+ GSS_API_INITIAL = 1
+ GSS_API_VERIFY = 2
+ GSS_API_FINISH = 3
+)
+
+type GSSAPIConfig struct {
+ AuthType int
+ KeyTabPath string
+ CCachePath string
+ KerberosConfigPath string
+ ServiceName string
+ Username string
+ Password string
+ Realm string
+ DisablePAFXFAST bool
+ BuildSpn BuildSpnFunc
+}
+
+type GSSAPIKerberosAuth struct {
+ Config *GSSAPIConfig
+ ticket messages.Ticket
+ encKey types.EncryptionKey
+ NewKerberosClientFunc func(config *GSSAPIConfig) (KerberosClient, error)
+ step int
+}
+
+type KerberosClient interface {
+ Login() error
+ GetServiceTicket(spn string) (messages.Ticket, types.EncryptionKey, error)
+ Domain() string
+ CName() types.PrincipalName
+ Destroy()
+}
+
+type BuildSpnFunc func(serviceName, host string) string
+
+// writePackage appends length in big endian before the payload, and sends it to kafka
+func (krbAuth *GSSAPIKerberosAuth) writePackage(broker *Broker, payload []byte) (int, error) {
+ length := uint64(len(payload))
+ size := length + 4 // 4 byte length header + payload
+ if size > math.MaxInt32 {
+ return 0, errors.New("payload too large, will overflow int32")
+ }
+ finalPackage := make([]byte, size)
+ copy(finalPackage[4:], payload)
+ binary.BigEndian.PutUint32(finalPackage, uint32(length))
+ bytes, err := broker.conn.Write(finalPackage)
+ if err != nil {
+ return bytes, err
+ }
+ return bytes, nil
+}
+
+// readPackage reads payload length (4 bytes) and then reads the payload into []byte
+func (krbAuth *GSSAPIKerberosAuth) readPackage(broker *Broker) ([]byte, int, error) {
+ bytesRead := 0
+ lengthInBytes := make([]byte, 4)
+ bytes, err := io.ReadFull(broker.conn, lengthInBytes)
+ if err != nil {
+ return nil, bytesRead, err
+ }
+ bytesRead += bytes
+ payloadLength := binary.BigEndian.Uint32(lengthInBytes)
+ payloadBytes := make([]byte, payloadLength) // buffer for read..
+ bytes, err = io.ReadFull(broker.conn, payloadBytes) // read bytes
+ if err != nil {
+ return payloadBytes, bytesRead, err
+ }
+ bytesRead += bytes
+ return payloadBytes, bytesRead, nil
+}
+
+func (krbAuth *GSSAPIKerberosAuth) newAuthenticatorChecksum() []byte {
+ a := make([]byte, 24)
+ flags := []int{gssapi.ContextFlagInteg, gssapi.ContextFlagConf}
+ binary.LittleEndian.PutUint32(a[:4], 16)
+ for _, i := range flags {
+ f := binary.LittleEndian.Uint32(a[20:24])
+ f |= uint32(i)
+ binary.LittleEndian.PutUint32(a[20:24], f)
+ }
+ return a
+}
+
+// Construct Kerberos AP_REQ package, conforming to RFC-4120
+// https://tools.ietf.org/html/rfc4120#page-84
+func (krbAuth *GSSAPIKerberosAuth) createKrb5Token(
+ domain string,
+ cname types.PrincipalName,
+ ticket messages.Ticket,
+ sessionKey types.EncryptionKey,
+) ([]byte, error) {
+ auth, err := types.NewAuthenticator(domain, cname)
+ if err != nil {
+ return nil, err
+ }
+ auth.Cksum = types.Checksum{
+ CksumType: chksumtype.GSSAPI,
+ Checksum: krbAuth.newAuthenticatorChecksum(),
+ }
+ APReq, err := messages.NewAPReq(
+ ticket,
+ sessionKey,
+ auth,
+ )
+ if err != nil {
+ return nil, err
+ }
+ aprBytes := make([]byte, 2)
+ binary.BigEndian.PutUint16(aprBytes, TOK_ID_KRB_AP_REQ)
+ tb, err := APReq.Marshal()
+ if err != nil {
+ return nil, err
+ }
+ aprBytes = append(aprBytes, tb...)
+ return aprBytes, nil
+}
+
+// Append the GSS-API header to the payload, conforming to RFC-2743
+// Section 3.1, Mechanism-Independent Token Format
+//
+// https://tools.ietf.org/html/rfc2743#page-81
+//
+// GSSAPIHeader + <specific mechanism payload>
+func (krbAuth *GSSAPIKerberosAuth) appendGSSAPIHeader(payload []byte) ([]byte, error) {
+ oidBytes, err := asn1.Marshal(gssapi.OIDKRB5.OID())
+ if err != nil {
+ return nil, err
+ }
+ tkoLengthBytes := asn1tools.MarshalLengthBytes(len(oidBytes) + len(payload))
+ GSSHeader := append([]byte{GSS_API_GENERIC_TAG}, tkoLengthBytes...)
+ GSSHeader = append(GSSHeader, oidBytes...)
+ GSSPackage := append(GSSHeader, payload...)
+ return GSSPackage, nil
+}
+
+func (krbAuth *GSSAPIKerberosAuth) initSecContext(
+ client KerberosClient,
+ bytes []byte,
+) ([]byte, error) {
+ switch krbAuth.step {
+ case GSS_API_INITIAL:
+ aprBytes, err := krbAuth.createKrb5Token(
+ client.Domain(),
+ client.CName(),
+ krbAuth.ticket,
+ krbAuth.encKey)
+ if err != nil {
+ return nil, err
+ }
+ krbAuth.step = GSS_API_VERIFY
+ return krbAuth.appendGSSAPIHeader(aprBytes)
+ case GSS_API_VERIFY:
+ wrapTokenReq := gssapi.WrapToken{}
+ if err := wrapTokenReq.Unmarshal(bytes, true); err != nil {
+ return nil, err
+ }
+ // Validate response.
+ isValid, err := wrapTokenReq.Verify(krbAuth.encKey, keyusage.GSSAPI_ACCEPTOR_SEAL)
+ if !isValid {
+ return nil, err
+ }
+
+ wrapTokenResponse, err := gssapi.NewInitiatorWrapToken(wrapTokenReq.Payload, krbAuth.encKey)
+ if err != nil {
+ return nil, err
+ }
+ krbAuth.step = GSS_API_FINISH
+ return wrapTokenResponse.Marshal()
+ }
+ return nil, nil
+}
+
+func (krbAuth *GSSAPIKerberosAuth) spn(broker *Broker) string {
+ host, _, _ := net.SplitHostPort(broker.addr)
+ var spn string
+ if krbAuth.Config.BuildSpn != nil {
+ spn = krbAuth.Config.BuildSpn(broker.conf.Net.SASL.GSSAPI.ServiceName, host)
+ } else {
+ spn = fmt.Sprintf("%s/%s", broker.conf.Net.SASL.GSSAPI.ServiceName, host)
+ }
+ return spn
+}
+
+// Login will use the given KerberosClient to login and get a ticket for the given spn.
+func (krbAuth *GSSAPIKerberosAuth) Login(
+ client KerberosClient,
+ spn string,
+) (*messages.Ticket, error) {
+ if err := client.Login(); err != nil {
+ Logger.Printf("Kerberos client login error: %s", err)
+ return nil, err
+ }
+
+ ticket, encKey, err := client.GetServiceTicket(spn)
+ if err != nil {
+ Logger.Printf("Kerberos service ticket error for %s: %s", spn, err)
+ return nil, err
+ }
+ krbAuth.ticket = ticket
+ krbAuth.encKey = encKey
+ krbAuth.step = GSS_API_INITIAL
+
+ return &ticket, nil
+}
+
+// Authorize performs the kerberos auth handshake for authorization
+func (krbAuth *GSSAPIKerberosAuth) Authorize(broker *Broker) error {
+ client, err := krbAuth.NewKerberosClientFunc(krbAuth.Config)
+ if err != nil {
+ Logger.Printf("Kerberos client initialization error: %s", err)
+ return err
+ }
+ defer client.Destroy()
+
+ ticket, err := krbAuth.Login(client, krbAuth.spn(broker))
+ if err != nil {
+ return err
+ }
+
+ principal := strings.Join(ticket.SName.NameString, "/") + "@" + ticket.Realm
+ var receivedBytes []byte
+
+ for {
+ packBytes, err := krbAuth.initSecContext(client, receivedBytes)
+ if err != nil {
+ Logger.Printf("Kerberos init error as %s: %s", principal, err)
+ return err
+ }
+
+ requestTime := time.Now()
+ bytesWritten, err := krbAuth.writePackage(broker, packBytes)
+ if err != nil {
+ Logger.Printf("Kerberos write error as %s: %s", principal, err)
+ return err
+ }
+ broker.updateOutgoingCommunicationMetrics(bytesWritten)
+
+ switch krbAuth.step {
+ case GSS_API_VERIFY:
+ var bytesRead int
+ receivedBytes, bytesRead, err = krbAuth.readPackage(broker)
+ requestLatency := time.Since(requestTime)
+ broker.updateIncomingCommunicationMetrics(bytesRead, requestLatency)
+ if err != nil {
+ Logger.Printf("Kerberos read error as %s: %s", principal, err)
+ return err
+ }
+ case GSS_API_FINISH:
+ return nil
+ }
+ }
+}
+
+// AuthorizeV2 performs the SASL v2 GSSAPI authentication with the Kafka broker.
+func (krbAuth *GSSAPIKerberosAuth) AuthorizeV2(
+ broker *Broker,
+ authSendReceiver func(authBytes []byte) (*SaslAuthenticateResponse, error),
+) error {
+ client, err := krbAuth.NewKerberosClientFunc(krbAuth.Config)
+ if err != nil {
+ Logger.Printf("Kerberos client initialization error: %s", err)
+ return err
+ }
+ defer client.Destroy()
+
+ ticket, err := krbAuth.Login(client, krbAuth.spn(broker))
+ if err != nil {
+ return err
+ }
+
+ principal := strings.Join(ticket.SName.NameString, "/") + "@" + ticket.Realm
+ var receivedBytes []byte
+
+ for {
+ token, err := krbAuth.initSecContext(client, receivedBytes)
+ if err != nil {
+ Logger.Printf("SASL Kerberos init error as %s: %s", principal, err)
+ return err
+ }
+
+ authResponse, err := authSendReceiver(token)
+ if err != nil {
+ Logger.Printf("SASL Kerberos authenticate error as %s: %s", principal, err)
+ return err
+ }
+
+ receivedBytes = authResponse.SaslAuthBytes
+
+ if krbAuth.step == GSS_API_FINISH {
+ return nil
+ }
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/heartbeat_request.go b/vendor/github.com/IBM/sarama/heartbeat_request.go
new file mode 100644
index 0000000..21b9971
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/heartbeat_request.go
@@ -0,0 +1,99 @@
+package sarama
+
+type HeartbeatRequest struct {
+ Version int16
+ GroupId string
+ GenerationId int32
+ MemberId string
+ GroupInstanceId *string
+}
+
+func (r *HeartbeatRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *HeartbeatRequest) encode(pe packetEncoder) error {
+ if err := pe.putString(r.GroupId); err != nil {
+ return err
+ }
+
+ pe.putInt32(r.GenerationId)
+
+ if err := pe.putString(r.MemberId); err != nil {
+ return err
+ }
+
+ if r.Version >= 3 {
+ if err := pe.putNullableString(r.GroupInstanceId); err != nil {
+ return err
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *HeartbeatRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ if r.GroupId, err = pd.getString(); err != nil {
+ return
+ }
+ if r.GenerationId, err = pd.getInt32(); err != nil {
+ return
+ }
+ if r.MemberId, err = pd.getString(); err != nil {
+ return
+ }
+ if r.Version >= 3 {
+ if r.GroupInstanceId, err = pd.getNullableString(); err != nil {
+ return
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *HeartbeatRequest) key() int16 {
+ return apiKeyHeartbeat
+}
+
+func (r *HeartbeatRequest) version() int16 {
+ return r.Version
+}
+
+func (r *HeartbeatRequest) headerVersion() int16 {
+ if r.Version >= 4 {
+ return 2
+ }
+ return 1
+}
+
+func (r *HeartbeatRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 4
+}
+
+func (r *HeartbeatRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *HeartbeatRequest) isFlexibleVersion(version int16) bool {
+ return version >= 4
+}
+
+func (r *HeartbeatRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 4:
+ return V2_4_0_0
+ case 3:
+ return V2_3_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_8_2_0
+ default:
+ return V2_3_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/heartbeat_response.go b/vendor/github.com/IBM/sarama/heartbeat_response.go
new file mode 100644
index 0000000..d753957
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/heartbeat_response.go
@@ -0,0 +1,87 @@
+package sarama
+
+import "time"
+
+type HeartbeatResponse struct {
+ Version int16
+ ThrottleTime int32
+ Err KError
+}
+
+func (r *HeartbeatResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *HeartbeatResponse) encode(pe packetEncoder) error {
+ if r.Version >= 1 {
+ pe.putInt32(r.ThrottleTime)
+ }
+ pe.putKError(r.Err)
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *HeartbeatResponse) decode(pd packetDecoder, version int16) error {
+ var err error
+ r.Version = version
+ if r.Version >= 1 {
+ if r.ThrottleTime, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+ r.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *HeartbeatResponse) key() int16 {
+ return apiKeyHeartbeat
+}
+
+func (r *HeartbeatResponse) version() int16 {
+ return r.Version
+}
+
+func (r *HeartbeatResponse) headerVersion() int16 {
+ if r.Version >= 4 {
+ return 1
+ }
+ return 0
+}
+
+func (r *HeartbeatResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 4
+}
+
+func (r *HeartbeatResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *HeartbeatResponse) isFlexibleVersion(version int16) bool {
+ return version >= 4
+}
+
+func (r *HeartbeatResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 4:
+ return V2_4_0_0
+ case 3:
+ return V2_3_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_8_2_0
+ default:
+ return V2_3_0_0
+ }
+}
+
+func (r *HeartbeatResponse) throttleTime() time.Duration {
+ return time.Duration(r.ThrottleTime) * time.Millisecond
+}
diff --git a/vendor/github.com/IBM/sarama/incremental_alter_configs_request.go b/vendor/github.com/IBM/sarama/incremental_alter_configs_request.go
new file mode 100644
index 0000000..e048684
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/incremental_alter_configs_request.go
@@ -0,0 +1,205 @@
+package sarama
+
+type IncrementalAlterConfigsOperation int8
+
+const (
+ IncrementalAlterConfigsOperationSet IncrementalAlterConfigsOperation = iota
+ IncrementalAlterConfigsOperationDelete
+ IncrementalAlterConfigsOperationAppend
+ IncrementalAlterConfigsOperationSubtract
+)
+
+// IncrementalAlterConfigsRequest is an incremental alter config request type
+type IncrementalAlterConfigsRequest struct {
+ Version int16
+ Resources []*IncrementalAlterConfigsResource
+ ValidateOnly bool
+}
+
+func (a *IncrementalAlterConfigsRequest) setVersion(v int16) {
+ a.Version = v
+}
+
+type IncrementalAlterConfigsResource struct {
+ Type ConfigResourceType
+ Name string
+ ConfigEntries map[string]IncrementalAlterConfigsEntry
+}
+
+type IncrementalAlterConfigsEntry struct {
+ Operation IncrementalAlterConfigsOperation
+ Value *string
+}
+
+func (a *IncrementalAlterConfigsRequest) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(a.Resources)); err != nil {
+ return err
+ }
+
+ for _, r := range a.Resources {
+ if err := r.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ pe.putBool(a.ValidateOnly)
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (a *IncrementalAlterConfigsRequest) decode(pd packetDecoder, version int16) error {
+ resourceCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ a.Resources = make([]*IncrementalAlterConfigsResource, resourceCount)
+ for i := range a.Resources {
+ r := &IncrementalAlterConfigsResource{}
+ err = r.decode(pd, version)
+ if err != nil {
+ return err
+ }
+ a.Resources[i] = r
+ }
+
+ validateOnly, err := pd.getBool()
+ if err != nil {
+ return err
+ }
+
+ a.ValidateOnly = validateOnly
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (a *IncrementalAlterConfigsResource) encode(pe packetEncoder) error {
+ pe.putInt8(int8(a.Type))
+
+ if err := pe.putString(a.Name); err != nil {
+ return err
+ }
+
+ if err := pe.putArrayLength(len(a.ConfigEntries)); err != nil {
+ return err
+ }
+
+ for name, e := range a.ConfigEntries {
+ if err := pe.putString(name); err != nil {
+ return err
+ }
+
+ if err := e.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (a *IncrementalAlterConfigsResource) decode(pd packetDecoder, version int16) error {
+ t, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ a.Type = ConfigResourceType(t)
+
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ a.Name = name
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if n > 0 {
+ a.ConfigEntries = make(map[string]IncrementalAlterConfigsEntry, n)
+ for i := 0; i < n; i++ {
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ var v IncrementalAlterConfigsEntry
+
+ if err := v.decode(pd, version); err != nil {
+ return err
+ }
+
+ a.ConfigEntries[name] = v
+ }
+ }
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (a *IncrementalAlterConfigsEntry) encode(pe packetEncoder) error {
+ pe.putInt8(int8(a.Operation))
+
+ if err := pe.putNullableString(a.Value); err != nil {
+ return err
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (a *IncrementalAlterConfigsEntry) decode(pd packetDecoder, version int16) error {
+ t, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ a.Operation = IncrementalAlterConfigsOperation(t)
+
+ s, err := pd.getNullableString()
+ if err != nil {
+ return err
+ }
+
+ a.Value = s
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (a *IncrementalAlterConfigsRequest) key() int16 {
+ return apiKeyIncrementalAlterConfigs
+}
+
+func (a *IncrementalAlterConfigsRequest) version() int16 {
+ return a.Version
+}
+
+func (a *IncrementalAlterConfigsRequest) headerVersion() int16 {
+ if a.Version >= 1 {
+ return 2
+ }
+ return 1
+}
+
+func (a *IncrementalAlterConfigsRequest) isValidVersion() bool {
+ return a.Version >= 0 && a.Version <= 1
+}
+
+func (a *IncrementalAlterConfigsRequest) isFlexible() bool {
+ return a.isFlexibleVersion(a.Version)
+}
+
+func (a *IncrementalAlterConfigsRequest) isFlexibleVersion(version int16) bool {
+ return version >= 1
+}
+
+func (a *IncrementalAlterConfigsRequest) requiredVersion() KafkaVersion {
+ switch a.Version {
+ case 1:
+ return V2_4_0_0
+ default:
+ return V2_3_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/incremental_alter_configs_response.go b/vendor/github.com/IBM/sarama/incremental_alter_configs_response.go
new file mode 100644
index 0000000..9333785
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/incremental_alter_configs_response.go
@@ -0,0 +1,94 @@
+package sarama
+
+import "time"
+
+// IncrementalAlterConfigsResponse is a response type for incremental alter config
+type IncrementalAlterConfigsResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ Resources []*AlterConfigsResourceResponse
+}
+
+func (a *IncrementalAlterConfigsResponse) setVersion(v int16) {
+ a.Version = v
+}
+
+func (a *IncrementalAlterConfigsResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(a.ThrottleTime)
+
+ if err := pe.putArrayLength(len(a.Resources)); err != nil {
+ return err
+ }
+
+ for _, v := range a.Resources {
+ if err := v.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (a *IncrementalAlterConfigsResponse) decode(pd packetDecoder, version int16) (err error) {
+ if a.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ responseCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ a.Resources = make([]*AlterConfigsResourceResponse, responseCount)
+
+ for i := range a.Resources {
+ a.Resources[i] = new(AlterConfigsResourceResponse)
+
+ if err := a.Resources[i].decode(pd, version); err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (a *IncrementalAlterConfigsResponse) key() int16 {
+ return apiKeyIncrementalAlterConfigs
+}
+
+func (a *IncrementalAlterConfigsResponse) version() int16 {
+ return a.Version
+}
+
+func (a *IncrementalAlterConfigsResponse) headerVersion() int16 {
+ if a.Version >= 1 {
+ return 1
+ }
+ return 0
+}
+
+func (a *IncrementalAlterConfigsResponse) isFlexible() bool {
+ return a.isFlexibleVersion(a.Version)
+}
+
+func (a *IncrementalAlterConfigsResponse) isFlexibleVersion(version int16) bool {
+ return version >= 1
+}
+
+func (a *IncrementalAlterConfigsResponse) isValidVersion() bool {
+ return a.Version >= 0 && a.Version <= 1
+}
+
+func (a *IncrementalAlterConfigsResponse) requiredVersion() KafkaVersion {
+ switch a.Version {
+ case 1:
+ return V2_4_0_0
+ default:
+ return V2_3_0_0
+ }
+}
+
+func (r *IncrementalAlterConfigsResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/init_producer_id_request.go b/vendor/github.com/IBM/sarama/init_producer_id_request.go
new file mode 100644
index 0000000..a5d6d26
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/init_producer_id_request.go
@@ -0,0 +1,99 @@
+package sarama
+
+import "time"
+
+type InitProducerIDRequest struct {
+ Version int16
+ TransactionalID *string
+ TransactionTimeout time.Duration
+ ProducerID int64
+ ProducerEpoch int16
+}
+
+func (i *InitProducerIDRequest) setVersion(v int16) {
+ i.Version = v
+}
+
+func (i *InitProducerIDRequest) encode(pe packetEncoder) error {
+ if err := pe.putNullableString(i.TransactionalID); err != nil {
+ return err
+ }
+ pe.putInt32(int32(i.TransactionTimeout / time.Millisecond))
+ if i.Version >= 3 {
+ pe.putInt64(i.ProducerID)
+ pe.putInt16(i.ProducerEpoch)
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (i *InitProducerIDRequest) decode(pd packetDecoder, version int16) (err error) {
+ i.Version = version
+ if i.TransactionalID, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ timeout, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ i.TransactionTimeout = time.Duration(timeout) * time.Millisecond
+ if i.Version >= 3 {
+ if i.ProducerID, err = pd.getInt64(); err != nil {
+ return err
+ }
+
+ if i.ProducerEpoch, err = pd.getInt16(); err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (i *InitProducerIDRequest) key() int16 {
+ return apiKeyInitProducerId
+}
+
+func (i *InitProducerIDRequest) version() int16 {
+ return i.Version
+}
+
+func (i *InitProducerIDRequest) headerVersion() int16 {
+ if i.Version >= 2 {
+ return 2
+ }
+
+ return 1
+}
+
+func (i *InitProducerIDRequest) isValidVersion() bool {
+ return i.Version >= 0 && i.Version <= 4
+}
+
+func (i *InitProducerIDRequest) isFlexible() bool {
+ return i.isFlexibleVersion(i.Version)
+}
+
+func (i *InitProducerIDRequest) isFlexibleVersion(version int16) bool {
+ return version >= 2
+}
+
+func (i *InitProducerIDRequest) requiredVersion() KafkaVersion {
+ switch i.Version {
+ case 4:
+ return V2_7_0_0
+ case 3:
+ return V2_5_0_0
+ case 2:
+ return V2_4_0_0
+ case 1:
+ return V2_0_0_0
+ case 0:
+ return V0_11_0_0
+ default:
+ return V2_7_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/init_producer_id_response.go b/vendor/github.com/IBM/sarama/init_producer_id_response.go
new file mode 100644
index 0000000..0e84c43
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/init_producer_id_response.go
@@ -0,0 +1,93 @@
+package sarama
+
+import "time"
+
+type InitProducerIDResponse struct {
+ ThrottleTime time.Duration
+ Err KError
+ Version int16
+ ProducerID int64
+ ProducerEpoch int16
+}
+
+func (i *InitProducerIDResponse) setVersion(v int16) {
+ i.Version = v
+}
+
+func (i *InitProducerIDResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(i.ThrottleTime)
+ pe.putKError(i.Err)
+ pe.putInt64(i.ProducerID)
+ pe.putInt16(i.ProducerEpoch)
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (i *InitProducerIDResponse) decode(pd packetDecoder, version int16) (err error) {
+ i.Version = version
+ if i.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+
+ i.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if i.ProducerID, err = pd.getInt64(); err != nil {
+ return err
+ }
+
+ if i.ProducerEpoch, err = pd.getInt16(); err != nil {
+ return err
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (i *InitProducerIDResponse) key() int16 {
+ return apiKeyInitProducerId
+}
+
+func (i *InitProducerIDResponse) version() int16 {
+ return i.Version
+}
+
+func (i *InitProducerIDResponse) headerVersion() int16 {
+ if i.Version >= 2 {
+ return 1
+ }
+ return 0
+}
+
+func (i *InitProducerIDResponse) isValidVersion() bool {
+ return i.Version >= 0 && i.Version <= 4
+}
+
+func (i *InitProducerIDResponse) isFlexible() bool {
+ return i.isFlexibleVersion(i.Version)
+}
+
+func (i *InitProducerIDResponse) isFlexibleVersion(version int16) bool {
+ return version >= 2
+}
+
+func (i *InitProducerIDResponse) requiredVersion() KafkaVersion {
+ switch i.Version {
+ case 4:
+ return V2_7_0_0
+ case 3:
+ return V2_5_0_0
+ case 2:
+ return V2_4_0_0
+ case 1:
+ return V2_0_0_0
+ default:
+ return V0_11_0_0
+ }
+}
+
+func (r *InitProducerIDResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/interceptors.go b/vendor/github.com/IBM/sarama/interceptors.go
new file mode 100644
index 0000000..d4dc23c
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/interceptors.go
@@ -0,0 +1,43 @@
+package sarama
+
+// ProducerInterceptor allows you to intercept (and possibly mutate) the records
+// received by the producer before they are published to the Kafka cluster.
+// https://cwiki.apache.org/confluence/display/KAFKA/KIP-42%3A+Add+Producer+and+Consumer+Interceptors#KIP42:AddProducerandConsumerInterceptors-Motivation
+type ProducerInterceptor interface {
+
+ // OnSend is called when the producer message is intercepted. Please avoid
+ // modifying the message until it's safe to do so, as this is _not_ a copy
+ // of the message.
+ OnSend(*ProducerMessage)
+}
+
+// ConsumerInterceptor allows you to intercept (and possibly mutate) the records
+// received by the consumer before they are sent to the messages channel.
+// https://cwiki.apache.org/confluence/display/KAFKA/KIP-42%3A+Add+Producer+and+Consumer+Interceptors#KIP42:AddProducerandConsumerInterceptors-Motivation
+type ConsumerInterceptor interface {
+
+ // OnConsume is called when the consumed message is intercepted. Please
+ // avoid modifying the message until it's safe to do so, as this is _not_ a
+ // copy of the message.
+ OnConsume(*ConsumerMessage)
+}
+
+func (msg *ProducerMessage) safelyApplyInterceptor(interceptor ProducerInterceptor) {
+ defer func() {
+ if r := recover(); r != nil {
+ Logger.Printf("Error when calling producer interceptor: %v, %v", interceptor, r)
+ }
+ }()
+
+ interceptor.OnSend(msg)
+}
+
+func (msg *ConsumerMessage) safelyApplyInterceptor(interceptor ConsumerInterceptor) {
+ defer func() {
+ if r := recover(); r != nil {
+ Logger.Printf("Error when calling consumer interceptor: %v, %v", interceptor, r)
+ }
+ }()
+
+ interceptor.OnConsume(msg)
+}
diff --git a/vendor/github.com/IBM/sarama/join_group_request.go b/vendor/github.com/IBM/sarama/join_group_request.go
new file mode 100644
index 0000000..e8cbc46
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/join_group_request.go
@@ -0,0 +1,234 @@
+package sarama
+
+type GroupProtocol struct {
+ // Name contains the protocol name.
+ Name string
+ // Metadata contains the protocol metadata.
+ Metadata []byte
+}
+
+func (p *GroupProtocol) decode(pd packetDecoder) (err error) {
+ p.Name, err = pd.getString()
+ if err != nil {
+ return err
+ }
+ p.Metadata, err = pd.getBytes()
+ if err != nil {
+ return err
+ }
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (p *GroupProtocol) encode(pe packetEncoder) (err error) {
+ if err := pe.putString(p.Name); err != nil {
+ return err
+ }
+ if err := pe.putBytes(p.Metadata); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+type JoinGroupRequest struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // GroupId contains the group identifier.
+ GroupId string
+ // SessionTimeout specifies that the coordinator should consider the consumer
+ // dead if it receives no heartbeat after this timeout in milliseconds.
+ SessionTimeout int32
+ // RebalanceTimeout contains the maximum time in milliseconds that the
+ // coordinator will wait for each member to rejoin when rebalancing the
+ // group.
+ RebalanceTimeout int32
+ // MemberId contains the member id assigned by the group coordinator.
+ MemberId string
+ // GroupInstanceId contains the unique identifier of the consumer instance
+ // provided by end user.
+ GroupInstanceId *string
+ // ProtocolType contains the unique name the for class of protocols
+ // implemented by the group we want to join.
+ ProtocolType string
+ // GroupProtocols contains the list of protocols that the member supports.
+ // deprecated; use OrderedGroupProtocols
+ GroupProtocols map[string][]byte
+ // OrderedGroupProtocols contains an ordered list of protocols that the member
+ // supports.
+ OrderedGroupProtocols []*GroupProtocol
+}
+
+func (r *JoinGroupRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *JoinGroupRequest) encode(pe packetEncoder) error {
+ if err := pe.putString(r.GroupId); err != nil {
+ return err
+ }
+ pe.putInt32(r.SessionTimeout)
+ if r.Version >= 1 {
+ pe.putInt32(r.RebalanceTimeout)
+ }
+ if err := pe.putString(r.MemberId); err != nil {
+ return err
+ }
+ if r.Version >= 5 {
+ if err := pe.putNullableString(r.GroupInstanceId); err != nil {
+ return err
+ }
+ }
+ if err := pe.putString(r.ProtocolType); err != nil {
+ return err
+ }
+
+ if len(r.GroupProtocols) > 0 {
+ if len(r.OrderedGroupProtocols) > 0 {
+ return PacketEncodingError{"cannot specify both GroupProtocols and OrderedGroupProtocols on JoinGroupRequest"}
+ }
+
+ if err := pe.putArrayLength(len(r.GroupProtocols)); err != nil {
+ return err
+ }
+ for name, metadata := range r.GroupProtocols {
+ if err := pe.putString(name); err != nil {
+ return err
+ }
+ if err := pe.putBytes(metadata); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+ } else {
+ if err := pe.putArrayLength(len(r.OrderedGroupProtocols)); err != nil {
+ return err
+ }
+ for _, protocol := range r.OrderedGroupProtocols {
+ if err := protocol.encode(pe); err != nil {
+ return err
+ }
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *JoinGroupRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+
+ if r.GroupId, err = pd.getString(); err != nil {
+ return
+ }
+
+ if r.SessionTimeout, err = pd.getInt32(); err != nil {
+ return
+ }
+
+ if version >= 1 {
+ if r.RebalanceTimeout, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+
+ if r.MemberId, err = pd.getString(); err != nil {
+ return
+ }
+
+ if version >= 5 {
+ if r.GroupInstanceId, err = pd.getNullableString(); err != nil {
+ return
+ }
+ }
+
+ if r.ProtocolType, err = pd.getString(); err != nil {
+ return
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if n == 0 {
+ return nil
+ }
+
+ r.GroupProtocols = make(map[string][]byte)
+ for i := 0; i < n; i++ {
+ protocol := &GroupProtocol{}
+ if err := protocol.decode(pd); err != nil {
+ return err
+ }
+ r.GroupProtocols[protocol.Name] = protocol.Metadata
+ r.OrderedGroupProtocols = append(r.OrderedGroupProtocols, protocol)
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *JoinGroupRequest) key() int16 {
+ return apiKeyJoinGroup
+}
+
+func (r *JoinGroupRequest) version() int16 {
+ return r.Version
+}
+
+func (r *JoinGroupRequest) headerVersion() int16 {
+ if r.Version >= 6 {
+ return 2
+ }
+ return 1
+}
+
+func (r *JoinGroupRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 6
+}
+
+func (r *JoinGroupRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *JoinGroupRequest) isFlexibleVersion(version int16) bool {
+ return version >= 6
+}
+
+func (r *JoinGroupRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 6:
+ return V2_4_0_0
+ case 5:
+ return V2_3_0_0
+ case 4:
+ return V2_2_0_0
+ case 3:
+ return V2_0_0_0
+ case 2:
+ return V0_11_0_0
+ case 1:
+ return V0_10_1_0
+ case 0:
+ return V0_10_0_0
+ default:
+ return V2_3_0_0
+ }
+}
+
+func (r *JoinGroupRequest) AddGroupProtocol(name string, metadata []byte) {
+ r.OrderedGroupProtocols = append(r.OrderedGroupProtocols, &GroupProtocol{
+ Name: name,
+ Metadata: metadata,
+ })
+}
+
+func (r *JoinGroupRequest) AddGroupProtocolMetadata(name string, metadata *ConsumerGroupMemberMetadata) error {
+ bin, err := encode(metadata, nil)
+ if err != nil {
+ return err
+ }
+
+ r.AddGroupProtocol(name, bin)
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/join_group_response.go b/vendor/github.com/IBM/sarama/join_group_response.go
new file mode 100644
index 0000000..bdabe0a
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/join_group_response.go
@@ -0,0 +1,211 @@
+package sarama
+
+import "time"
+
+type JoinGroupResponse struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // ThrottleTime contains the duration for which the request was throttled due
+ // to a quota violation, or zero if the request did not violate any quota.
+ ThrottleTime int32
+ // Err contains the error code, or 0 if there was no error.
+ Err KError
+ // GenerationId contains the generation ID of the group.
+ GenerationId int32
+ // GroupProtocol contains the group protocol selected by the coordinator.
+ GroupProtocol string
+ // LeaderId contains the leader of the group.
+ LeaderId string
+ // MemberId contains the member ID assigned by the group coordinator.
+ MemberId string
+ // Members contains the per-group-member information.
+ Members []GroupMember
+}
+
+func (r *JoinGroupResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+type GroupMember struct {
+ // MemberId contains the group member ID.
+ MemberId string
+ // GroupInstanceId contains the unique identifier of the consumer instance
+ // provided by end user.
+ GroupInstanceId *string
+ // Metadata contains the group member metadata.
+ Metadata []byte
+}
+
+func (r *JoinGroupResponse) GetMembers() (map[string]ConsumerGroupMemberMetadata, error) {
+ members := make(map[string]ConsumerGroupMemberMetadata, len(r.Members))
+ for _, member := range r.Members {
+ meta := new(ConsumerGroupMemberMetadata)
+ if err := decode(member.Metadata, meta, nil); err != nil {
+ return nil, err
+ }
+ members[member.MemberId] = *meta
+ }
+ return members, nil
+}
+
+func (r *JoinGroupResponse) encode(pe packetEncoder) error {
+ if r.Version >= 2 {
+ pe.putInt32(r.ThrottleTime)
+ }
+ pe.putKError(r.Err)
+ pe.putInt32(r.GenerationId)
+
+ if err := pe.putString(r.GroupProtocol); err != nil {
+ return err
+ }
+ if err := pe.putString(r.LeaderId); err != nil {
+ return err
+ }
+ if err := pe.putString(r.MemberId); err != nil {
+ return err
+ }
+
+ if err := pe.putArrayLength(len(r.Members)); err != nil {
+ return err
+ }
+
+ for _, member := range r.Members {
+ if err := pe.putString(member.MemberId); err != nil {
+ return err
+ }
+ if r.Version >= 5 {
+ if err := pe.putNullableString(member.GroupInstanceId); err != nil {
+ return err
+ }
+ }
+ if err := pe.putBytes(member.Metadata); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *JoinGroupResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+
+ if version >= 2 {
+ if r.ThrottleTime, err = pd.getInt32(); err != nil {
+ return
+ }
+ }
+
+ r.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if r.GenerationId, err = pd.getInt32(); err != nil {
+ return
+ }
+
+ if r.GroupProtocol, err = pd.getString(); err != nil {
+ return
+ }
+
+ if r.LeaderId, err = pd.getString(); err != nil {
+ return
+ }
+
+ if r.MemberId, err = pd.getString(); err != nil {
+ return
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if n == 0 {
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+ }
+
+ r.Members = make([]GroupMember, n)
+ for i := 0; i < n; i++ {
+ memberId, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ var groupInstanceId *string = nil
+ if r.Version >= 5 {
+ groupInstanceId, err = pd.getNullableString()
+ if err != nil {
+ return err
+ }
+ }
+
+ memberMetadata, err := pd.getBytes()
+ if err != nil {
+ return err
+ }
+
+ r.Members[i] = GroupMember{MemberId: memberId, GroupInstanceId: groupInstanceId, Metadata: memberMetadata}
+
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *JoinGroupResponse) key() int16 {
+ return apiKeyJoinGroup
+}
+
+func (r *JoinGroupResponse) version() int16 {
+ return r.Version
+}
+
+func (r *JoinGroupResponse) headerVersion() int16 {
+ if r.Version >= 6 {
+ return 1
+ }
+ return 0
+}
+
+func (r *JoinGroupResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 6
+}
+
+func (r *JoinGroupResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *JoinGroupResponse) isFlexibleVersion(version int16) bool {
+ return version >= 6
+}
+
+func (r *JoinGroupResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 6:
+ return V2_4_0_0
+ case 5:
+ return V2_3_0_0
+ case 4:
+ return V2_2_0_0
+ case 3:
+ return V2_0_0_0
+ case 2:
+ return V0_11_0_0
+ case 1:
+ return V0_10_1_0
+ case 0:
+ return V0_10_0_0
+ default:
+ return V2_3_0_0
+ }
+}
+
+func (r *JoinGroupResponse) throttleTime() time.Duration {
+ return time.Duration(r.ThrottleTime) * time.Millisecond
+}
diff --git a/vendor/github.com/IBM/sarama/kerberos_client.go b/vendor/github.com/IBM/sarama/kerberos_client.go
new file mode 100644
index 0000000..2891268
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/kerberos_client.go
@@ -0,0 +1,57 @@
+package sarama
+
+import (
+ krb5client "github.com/jcmturner/gokrb5/v8/client"
+ krb5config "github.com/jcmturner/gokrb5/v8/config"
+ "github.com/jcmturner/gokrb5/v8/credentials"
+ "github.com/jcmturner/gokrb5/v8/keytab"
+ "github.com/jcmturner/gokrb5/v8/types"
+)
+
+type KerberosGoKrb5Client struct {
+ krb5client.Client
+}
+
+func (c *KerberosGoKrb5Client) Domain() string {
+ return c.Credentials.Domain()
+}
+
+func (c *KerberosGoKrb5Client) CName() types.PrincipalName {
+ return c.Credentials.CName()
+}
+
+// NewKerberosClient creates kerberos client used to obtain TGT and TGS tokens.
+// It uses pure go Kerberos 5 solution (RFC-4121 and RFC-4120).
+// uses gokrb5 library underlying which is a pure go kerberos client with some GSS-API capabilities.
+func NewKerberosClient(config *GSSAPIConfig) (KerberosClient, error) {
+ cfg, err := krb5config.Load(config.KerberosConfigPath)
+ if err != nil {
+ return nil, err
+ }
+ return createClient(config, cfg)
+}
+
+func createClient(config *GSSAPIConfig, cfg *krb5config.Config) (KerberosClient, error) {
+ var client *krb5client.Client
+ switch config.AuthType {
+ case KRB5_KEYTAB_AUTH:
+ kt, err := keytab.Load(config.KeyTabPath)
+ if err != nil {
+ return nil, err
+ }
+ client = krb5client.NewWithKeytab(config.Username, config.Realm, kt, cfg, krb5client.DisablePAFXFAST(config.DisablePAFXFAST))
+ case KRB5_CCACHE_AUTH:
+ cc, err := credentials.LoadCCache(config.CCachePath)
+ if err != nil {
+ return nil, err
+ }
+ client, err = krb5client.NewFromCCache(cc, cfg, krb5client.DisablePAFXFAST(config.DisablePAFXFAST))
+ if err != nil {
+ return nil, err
+ }
+ default:
+ client = krb5client.NewWithPassword(config.Username,
+ config.Realm, config.Password, cfg, krb5client.DisablePAFXFAST(config.DisablePAFXFAST))
+ }
+ return &KerberosGoKrb5Client{*client}, nil
+}
diff --git a/vendor/github.com/IBM/sarama/leave_group_request.go b/vendor/github.com/IBM/sarama/leave_group_request.go
new file mode 100644
index 0000000..377bdc5
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/leave_group_request.go
@@ -0,0 +1,125 @@
+package sarama
+
+type MemberIdentity struct {
+ MemberId string
+ GroupInstanceId *string
+}
+
+type LeaveGroupRequest struct {
+ Version int16
+ GroupId string
+ MemberId string // Removed in Version 3
+ Members []MemberIdentity // Added in Version 3
+}
+
+func (r *LeaveGroupRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *LeaveGroupRequest) encode(pe packetEncoder) error {
+ if err := pe.putString(r.GroupId); err != nil {
+ return err
+ }
+ if r.Version < 3 {
+ if err := pe.putString(r.MemberId); err != nil {
+ return err
+ }
+ }
+ if r.Version >= 3 {
+ if err := pe.putArrayLength(len(r.Members)); err != nil {
+ return err
+ }
+ for _, member := range r.Members {
+ if err := pe.putString(member.MemberId); err != nil {
+ return err
+ }
+ if err := pe.putNullableString(member.GroupInstanceId); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *LeaveGroupRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ if r.GroupId, err = pd.getString(); err != nil {
+ return
+ }
+ if r.Version < 3 {
+ if r.MemberId, err = pd.getString(); err != nil {
+ return
+ }
+ }
+ if r.Version >= 3 {
+ memberCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ r.Members = make([]MemberIdentity, memberCount)
+ for i := 0; i < memberCount; i++ {
+ memberIdentity := MemberIdentity{}
+ if memberIdentity.MemberId, err = pd.getString(); err != nil {
+ return err
+ }
+ if memberIdentity.GroupInstanceId, err = pd.getNullableString(); err != nil {
+ return err
+ }
+ r.Members[i] = memberIdentity
+ _, err = pd.getEmptyTaggedFieldArray()
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *LeaveGroupRequest) key() int16 {
+ return apiKeyLeaveGroup
+}
+
+func (r *LeaveGroupRequest) version() int16 {
+ return r.Version
+}
+
+func (r *LeaveGroupRequest) headerVersion() int16 {
+ if r.Version >= 4 {
+ return 2
+ }
+ return 1
+}
+
+func (r *LeaveGroupRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 4
+}
+
+func (r *LeaveGroupRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *LeaveGroupRequest) isFlexibleVersion(version int16) bool {
+ return version >= 4
+}
+
+func (r *LeaveGroupRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 4:
+ return V2_4_0_0
+ case 3:
+ return V2_4_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_9_0_0
+ default:
+ return V2_4_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/leave_group_response.go b/vendor/github.com/IBM/sarama/leave_group_response.go
new file mode 100644
index 0000000..5700992
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/leave_group_response.go
@@ -0,0 +1,129 @@
+package sarama
+
+import "time"
+
+type MemberResponse struct {
+ MemberId string
+ GroupInstanceId *string
+ Err KError
+}
+type LeaveGroupResponse struct {
+ Version int16
+ ThrottleTime int32
+ Err KError
+ Members []MemberResponse
+}
+
+func (r *LeaveGroupResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *LeaveGroupResponse) encode(pe packetEncoder) error {
+ if r.Version >= 1 {
+ pe.putInt32(r.ThrottleTime)
+ }
+ pe.putKError(r.Err)
+ if r.Version >= 3 {
+ if err := pe.putArrayLength(len(r.Members)); err != nil {
+ return err
+ }
+ for _, member := range r.Members {
+ if err := pe.putString(member.MemberId); err != nil {
+ return err
+ }
+ if err := pe.putNullableString(member.GroupInstanceId); err != nil {
+ return err
+ }
+ pe.putKError(member.Err)
+ pe.putEmptyTaggedFieldArray()
+ }
+ }
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *LeaveGroupResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ if r.Version >= 1 {
+ if r.ThrottleTime, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+ r.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if r.Version >= 3 {
+ membersLen, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ r.Members = make([]MemberResponse, membersLen)
+ for i := 0; i < len(r.Members); i++ {
+ if r.Members[i].MemberId, err = pd.getString(); err != nil {
+ return err
+ }
+ if r.Members[i].GroupInstanceId, err = pd.getNullableString(); err != nil {
+ return err
+ }
+ if r.Members[i].Err, err = pd.getKError(); err != nil {
+ return err
+ }
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *LeaveGroupResponse) key() int16 {
+ return apiKeyLeaveGroup
+}
+
+func (r *LeaveGroupResponse) version() int16 {
+ return r.Version
+}
+
+func (r *LeaveGroupResponse) headerVersion() int16 {
+ if r.Version >= 4 {
+ return 1
+ }
+ return 0
+}
+
+func (r *LeaveGroupResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 4
+}
+
+func (r *LeaveGroupResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *LeaveGroupResponse) isFlexibleVersion(version int16) bool {
+ return version >= 4
+}
+
+func (r *LeaveGroupResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 4:
+ return V2_4_0_0
+ case 3:
+ return V2_4_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_9_0_0
+ default:
+ return V2_4_0_0
+ }
+}
+
+func (r *LeaveGroupResponse) throttleTime() time.Duration {
+ return time.Duration(r.ThrottleTime) * time.Millisecond
+}
diff --git a/vendor/github.com/IBM/sarama/length_field.go b/vendor/github.com/IBM/sarama/length_field.go
new file mode 100644
index 0000000..7d864f6
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/length_field.go
@@ -0,0 +1,99 @@
+package sarama
+
+import (
+ "encoding/binary"
+ "sync"
+)
+
+// LengthField implements the PushEncoder and PushDecoder interfaces for calculating 4-byte lengths.
+type lengthField struct {
+ startOffset int
+ length int32
+}
+
+var lengthFieldPool = sync.Pool{}
+
+func acquireLengthField() *lengthField {
+ val := lengthFieldPool.Get()
+ if val != nil {
+ return val.(*lengthField)
+ }
+ return &lengthField{}
+}
+
+func releaseLengthField(m *lengthField) {
+ lengthFieldPool.Put(m)
+}
+
+func (l *lengthField) decode(pd packetDecoder) error {
+ var err error
+ l.length, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+ if l.length > int32(pd.remaining()) {
+ return ErrInsufficientData
+ }
+ return nil
+}
+
+func (l *lengthField) saveOffset(in int) {
+ l.startOffset = in
+}
+
+func (l *lengthField) reserveLength() int {
+ return 4
+}
+
+func (l *lengthField) run(curOffset int, buf []byte) error {
+ binary.BigEndian.PutUint32(buf[l.startOffset:], uint32(curOffset-l.startOffset-4))
+ return nil
+}
+
+func (l *lengthField) check(curOffset int, buf []byte) error {
+ if int32(curOffset-l.startOffset-4) != l.length {
+ return PacketDecodingError{"length field invalid"}
+ }
+
+ return nil
+}
+
+type varintLengthField struct {
+ startOffset int
+ length int64
+}
+
+func (l *varintLengthField) decode(pd packetDecoder) error {
+ var err error
+ l.length, err = pd.getVarint()
+ return err
+}
+
+func (l *varintLengthField) saveOffset(in int) {
+ l.startOffset = in
+}
+
+func (l *varintLengthField) adjustLength(currOffset int) int {
+ oldFieldSize := l.reserveLength()
+ l.length = int64(currOffset - l.startOffset - oldFieldSize)
+
+ return l.reserveLength() - oldFieldSize
+}
+
+func (l *varintLengthField) reserveLength() int {
+ var tmp [binary.MaxVarintLen64]byte
+ return binary.PutVarint(tmp[:], l.length)
+}
+
+func (l *varintLengthField) run(curOffset int, buf []byte) error {
+ binary.PutVarint(buf[l.startOffset:], l.length)
+ return nil
+}
+
+func (l *varintLengthField) check(curOffset int, buf []byte) error {
+ if int64(curOffset-l.startOffset-l.reserveLength()) != l.length {
+ return PacketDecodingError{"length field invalid"}
+ }
+
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/list_groups_request.go b/vendor/github.com/IBM/sarama/list_groups_request.go
new file mode 100644
index 0000000..b02e331
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/list_groups_request.go
@@ -0,0 +1,100 @@
+package sarama
+
+type ListGroupsRequest struct {
+ Version int16
+ StatesFilter []string // version 4 or later
+ TypesFilter []string // version 5 or later
+}
+
+func (r *ListGroupsRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *ListGroupsRequest) encode(pe packetEncoder) error {
+ if r.Version >= 4 {
+ if err := pe.putArrayLength(len(r.StatesFilter)); err != nil {
+ return err
+ }
+ for _, filter := range r.StatesFilter {
+ err := pe.putString(filter)
+ if err != nil {
+ return err
+ }
+ }
+ if r.Version >= 5 {
+ if err := pe.putArrayLength(len(r.TypesFilter)); err != nil {
+ return err
+ }
+ for _, filter := range r.TypesFilter {
+ err := pe.putString(filter)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ }
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *ListGroupsRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ if r.Version >= 4 {
+ if r.StatesFilter, err = pd.getStringArray(); err != nil {
+ return err
+ }
+ if r.Version >= 5 {
+ if r.TypesFilter, err = pd.getStringArray(); err != nil {
+ return err
+ }
+ }
+ }
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *ListGroupsRequest) key() int16 {
+ return apiKeyListGroups
+}
+
+func (r *ListGroupsRequest) version() int16 {
+ return r.Version
+}
+
+func (r *ListGroupsRequest) headerVersion() int16 {
+ if r.Version >= 3 {
+ return 2
+ }
+ return 1
+}
+
+func (r *ListGroupsRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 5
+}
+
+func (r *ListGroupsRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *ListGroupsRequest) isFlexibleVersion(version int16) bool {
+ return version >= 3
+}
+
+func (r *ListGroupsRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 5:
+ return V3_8_0_0
+ case 4:
+ return V2_6_0_0
+ case 3:
+ return V2_4_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_9_0_0
+ default:
+ return V2_6_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/list_groups_response.go b/vendor/github.com/IBM/sarama/list_groups_response.go
new file mode 100644
index 0000000..7e3a55f
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/list_groups_response.go
@@ -0,0 +1,165 @@
+package sarama
+
+type ListGroupsResponse struct {
+ Version int16
+ ThrottleTime int32
+ Err KError
+ Groups map[string]string
+ GroupsData map[string]GroupData // version 4 or later
+}
+
+func (r *ListGroupsResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+type GroupData struct {
+ GroupState string // version 4 or later
+ GroupType string // version 5 or later
+}
+
+func (r *ListGroupsResponse) encode(pe packetEncoder) error {
+ if r.Version >= 1 {
+ pe.putInt32(r.ThrottleTime)
+ }
+
+ pe.putKError(r.Err)
+
+ if err := pe.putArrayLength(len(r.Groups)); err != nil {
+ return err
+ }
+ for groupId, protocolType := range r.Groups {
+ if err := pe.putString(groupId); err != nil {
+ return err
+ }
+ if err := pe.putString(protocolType); err != nil {
+ return err
+ }
+ if r.Version >= 4 {
+ groupData := r.GroupsData[groupId]
+ if err := pe.putString(groupData.GroupState); err != nil {
+ return err
+ }
+ }
+
+ if r.Version >= 5 {
+ groupData := r.GroupsData[groupId]
+ if err := pe.putString(groupData.GroupType); err != nil {
+ return err
+ }
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *ListGroupsResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ if r.Version >= 1 {
+ if r.ThrottleTime, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+
+ r.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ for i := 0; i < n; i++ {
+ if i == 0 {
+ r.Groups = make(map[string]string)
+ if r.Version >= 4 {
+ r.GroupsData = make(map[string]GroupData)
+ }
+ }
+
+ var groupId, protocolType string
+ groupId, err = pd.getString()
+ if err != nil {
+ return err
+ }
+ protocolType, err = pd.getString()
+ if err != nil {
+ return err
+ }
+
+ r.Groups[groupId] = protocolType
+
+ if r.Version >= 4 {
+ var groupData GroupData
+ groupState, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ groupData.GroupState = groupState
+ if r.Version >= 5 {
+ groupType, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ groupData.GroupType = groupType
+ }
+ r.GroupsData[groupId] = groupData
+ }
+
+ if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *ListGroupsResponse) key() int16 {
+ return apiKeyListGroups
+}
+
+func (r *ListGroupsResponse) version() int16 {
+ return r.Version
+}
+
+func (r *ListGroupsResponse) headerVersion() int16 {
+ if r.Version >= 3 {
+ return 1
+ }
+ return 0
+}
+
+func (r *ListGroupsResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 5
+}
+
+func (r *ListGroupsResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *ListGroupsResponse) isFlexibleVersion(version int16) bool {
+ return version >= 3
+}
+
+func (r *ListGroupsResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 5:
+ return V3_8_0_0
+ case 4:
+ return V2_6_0_0
+ case 3:
+ return V2_4_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_9_0_0
+ default:
+ return V2_6_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/list_partition_reassignments_request.go b/vendor/github.com/IBM/sarama/list_partition_reassignments_request.go
new file mode 100644
index 0000000..25b1b29
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/list_partition_reassignments_request.go
@@ -0,0 +1,113 @@
+package sarama
+
+type ListPartitionReassignmentsRequest struct {
+ TimeoutMs int32
+ blocks map[string][]int32
+ Version int16
+}
+
+func (r *ListPartitionReassignmentsRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *ListPartitionReassignmentsRequest) encode(pe packetEncoder) error {
+ pe.putInt32(r.TimeoutMs)
+
+ if err := pe.putArrayLength(len(r.blocks)); err != nil {
+ return err
+ }
+
+ for topic, partitions := range r.blocks {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+
+ if err := pe.putInt32Array(partitions); err != nil {
+ return err
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+
+ return nil
+}
+
+func (r *ListPartitionReassignmentsRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+
+ if r.TimeoutMs, err = pd.getInt32(); err != nil {
+ return err
+ }
+
+ topicCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if topicCount > 0 {
+ r.blocks = make(map[string][]int32)
+ for i := 0; i < topicCount; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ partitionCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ r.blocks[topic] = make([]int32, partitionCount)
+ for j := 0; j < partitionCount; j++ {
+ partition, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ r.blocks[topic][j] = partition
+ }
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *ListPartitionReassignmentsRequest) key() int16 {
+ return apiKeyListPartitionReassignments
+}
+
+func (r *ListPartitionReassignmentsRequest) version() int16 {
+ return r.Version
+}
+
+func (r *ListPartitionReassignmentsRequest) headerVersion() int16 {
+ return 2
+}
+
+func (r *ListPartitionReassignmentsRequest) isValidVersion() bool {
+ return r.Version == 0
+}
+
+func (r *ListPartitionReassignmentsRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *ListPartitionReassignmentsRequest) isFlexibleVersion(version int16) bool {
+ return version >= 0
+}
+
+func (r *ListPartitionReassignmentsRequest) requiredVersion() KafkaVersion {
+ return V2_4_0_0
+}
+
+func (r *ListPartitionReassignmentsRequest) AddBlock(topic string, partitionIDs []int32) {
+ if r.blocks == nil {
+ r.blocks = make(map[string][]int32)
+ }
+
+ if r.blocks[topic] == nil {
+ r.blocks[topic] = partitionIDs
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/list_partition_reassignments_response.go b/vendor/github.com/IBM/sarama/list_partition_reassignments_response.go
new file mode 100644
index 0000000..8ade1e4
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/list_partition_reassignments_response.go
@@ -0,0 +1,188 @@
+package sarama
+
+import "time"
+
+type PartitionReplicaReassignmentsStatus struct {
+ Replicas []int32
+ AddingReplicas []int32
+ RemovingReplicas []int32
+}
+
+func (b *PartitionReplicaReassignmentsStatus) encode(pe packetEncoder) error {
+ if err := pe.putInt32Array(b.Replicas); err != nil {
+ return err
+ }
+ if err := pe.putInt32Array(b.AddingReplicas); err != nil {
+ return err
+ }
+ if err := pe.putInt32Array(b.RemovingReplicas); err != nil {
+ return err
+ }
+
+ pe.putEmptyTaggedFieldArray()
+
+ return nil
+}
+
+func (b *PartitionReplicaReassignmentsStatus) decode(pd packetDecoder) (err error) {
+ if b.Replicas, err = pd.getInt32Array(); err != nil {
+ return err
+ }
+
+ if b.AddingReplicas, err = pd.getInt32Array(); err != nil {
+ return err
+ }
+
+ if b.RemovingReplicas, err = pd.getInt32Array(); err != nil {
+ return err
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+type ListPartitionReassignmentsResponse struct {
+ Version int16
+ ThrottleTimeMs int32
+ ErrorCode KError
+ ErrorMessage *string
+ TopicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus
+}
+
+func (r *ListPartitionReassignmentsResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *ListPartitionReassignmentsResponse) AddBlock(topic string, partition int32, replicas, addingReplicas, removingReplicas []int32) {
+ if r.TopicStatus == nil {
+ r.TopicStatus = make(map[string]map[int32]*PartitionReplicaReassignmentsStatus)
+ }
+ partitions := r.TopicStatus[topic]
+ if partitions == nil {
+ partitions = make(map[int32]*PartitionReplicaReassignmentsStatus)
+ r.TopicStatus[topic] = partitions
+ }
+
+ partitions[partition] = &PartitionReplicaReassignmentsStatus{Replicas: replicas, AddingReplicas: addingReplicas, RemovingReplicas: removingReplicas}
+}
+
+func (r *ListPartitionReassignmentsResponse) encode(pe packetEncoder) error {
+ pe.putInt32(r.ThrottleTimeMs)
+ pe.putKError(r.ErrorCode)
+ if err := pe.putNullableString(r.ErrorMessage); err != nil {
+ return err
+ }
+
+ if err := pe.putArrayLength(len(r.TopicStatus)); err != nil {
+ return err
+ }
+ for topic, partitions := range r.TopicStatus {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putArrayLength(len(partitions)); err != nil {
+ return err
+ }
+ for partition, block := range partitions {
+ pe.putInt32(partition)
+
+ if err := block.encode(pe); err != nil {
+ return err
+ }
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ pe.putEmptyTaggedFieldArray()
+
+ return nil
+}
+
+func (r *ListPartitionReassignmentsResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+
+ if r.ThrottleTimeMs, err = pd.getInt32(); err != nil {
+ return err
+ }
+
+ r.ErrorCode, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if r.ErrorMessage, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ numTopics, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.TopicStatus = make(map[string]map[int32]*PartitionReplicaReassignmentsStatus, numTopics)
+ for i := 0; i < numTopics; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ ongoingPartitionReassignments, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.TopicStatus[topic] = make(map[int32]*PartitionReplicaReassignmentsStatus, ongoingPartitionReassignments)
+
+ for j := 0; j < ongoingPartitionReassignments; j++ {
+ partition, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ block := &PartitionReplicaReassignmentsStatus{}
+ if err := block.decode(pd); err != nil {
+ return err
+ }
+ r.TopicStatus[topic][partition] = block
+ }
+
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *ListPartitionReassignmentsResponse) key() int16 {
+ return apiKeyListPartitionReassignments
+}
+
+func (r *ListPartitionReassignmentsResponse) version() int16 {
+ return r.Version
+}
+
+func (r *ListPartitionReassignmentsResponse) headerVersion() int16 {
+ return 1
+}
+
+func (r *ListPartitionReassignmentsResponse) isValidVersion() bool {
+ return r.Version == 0
+}
+
+func (r *ListPartitionReassignmentsResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *ListPartitionReassignmentsResponse) isFlexibleVersion(version int16) bool {
+ return version >= 0
+}
+
+func (r *ListPartitionReassignmentsResponse) requiredVersion() KafkaVersion {
+ return V2_4_0_0
+}
+
+func (r *ListPartitionReassignmentsResponse) throttleTime() time.Duration {
+ return time.Duration(r.ThrottleTimeMs) * time.Millisecond
+}
diff --git a/vendor/github.com/IBM/sarama/message.go b/vendor/github.com/IBM/sarama/message.go
new file mode 100644
index 0000000..c6f35a3
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/message.go
@@ -0,0 +1,190 @@
+package sarama
+
+import (
+ "fmt"
+ "time"
+)
+
+const (
+ // CompressionNone no compression
+ CompressionNone CompressionCodec = iota
+ // CompressionGZIP compression using GZIP
+ CompressionGZIP
+ // CompressionSnappy compression using snappy
+ CompressionSnappy
+ // CompressionLZ4 compression using LZ4
+ CompressionLZ4
+ // CompressionZSTD compression using ZSTD
+ CompressionZSTD
+
+ // The lowest 3 bits contain the compression codec used for the message
+ compressionCodecMask int8 = 0x07
+
+ // Bit 3 set for "LogAppend" timestamps
+ timestampTypeMask = 0x08
+
+ // CompressionLevelDefault is the constant to use in CompressionLevel
+ // to have the default compression level for any codec. The value is picked
+ // that we don't use any existing compression levels.
+ CompressionLevelDefault = -1000
+)
+
+// CompressionCodec represents the various compression codecs recognized by Kafka in messages.
+type CompressionCodec int8
+
+func (cc CompressionCodec) String() string {
+ return []string{
+ "none",
+ "gzip",
+ "snappy",
+ "lz4",
+ "zstd",
+ }[int(cc)]
+}
+
+// UnmarshalText returns a CompressionCodec from its string representation.
+func (cc *CompressionCodec) UnmarshalText(text []byte) error {
+ codecs := map[string]CompressionCodec{
+ "none": CompressionNone,
+ "gzip": CompressionGZIP,
+ "snappy": CompressionSnappy,
+ "lz4": CompressionLZ4,
+ "zstd": CompressionZSTD,
+ }
+ codec, ok := codecs[string(text)]
+ if !ok {
+ return fmt.Errorf("cannot parse %q as a compression codec", string(text))
+ }
+ *cc = codec
+ return nil
+}
+
+// MarshalText transforms a CompressionCodec into its string representation.
+func (cc CompressionCodec) MarshalText() ([]byte, error) {
+ return []byte(cc.String()), nil
+}
+
+// Message is a kafka message type
+type Message struct {
+ Codec CompressionCodec // codec used to compress the message contents
+ CompressionLevel int // compression level
+ LogAppendTime bool // the used timestamp is LogAppendTime
+ Key []byte // the message key, may be nil
+ Value []byte // the message contents
+ Set *MessageSet // the message set a message might wrap
+ Version int8 // v1 requires Kafka 0.10
+ Timestamp time.Time // the timestamp of the message (version 1+ only)
+
+ compressedCache []byte
+ compressedSize int // used for computing the compression ratio metrics
+}
+
+func (m *Message) encode(pe packetEncoder) error {
+ pe.push(newCRC32Field(crcIEEE))
+
+ pe.putInt8(m.Version)
+
+ attributes := int8(m.Codec) & compressionCodecMask
+ if m.LogAppendTime {
+ attributes |= timestampTypeMask
+ }
+ pe.putInt8(attributes)
+
+ if m.Version >= 1 {
+ if err := (Timestamp{&m.Timestamp}).encode(pe); err != nil {
+ return err
+ }
+ }
+
+ err := pe.putBytes(m.Key)
+ if err != nil {
+ return err
+ }
+
+ var payload []byte
+
+ if m.compressedCache != nil {
+ payload = m.compressedCache
+ m.compressedCache = nil
+ } else if m.Value != nil {
+ payload, err = compress(m.Codec, m.CompressionLevel, m.Value)
+ if err != nil {
+ return err
+ }
+ m.compressedCache = payload
+ // Keep in mind the compressed payload size for metric gathering
+ m.compressedSize = len(payload)
+ }
+
+ if err = pe.putBytes(payload); err != nil {
+ return err
+ }
+
+ return pe.pop()
+}
+
+func (m *Message) decode(pd packetDecoder) (err error) {
+ crc32Decoder := acquireCrc32Field(crcIEEE)
+ defer releaseCrc32Field(crc32Decoder)
+
+ err = pd.push(crc32Decoder)
+ if err != nil {
+ return err
+ }
+
+ m.Version, err = pd.getInt8()
+ if err != nil {
+ return err
+ }
+
+ if m.Version > 1 {
+ return PacketDecodingError{fmt.Sprintf("unknown magic byte (%v)", m.Version)}
+ }
+
+ attribute, err := pd.getInt8()
+ if err != nil {
+ return err
+ }
+ m.Codec = CompressionCodec(attribute & compressionCodecMask)
+ m.LogAppendTime = attribute×tampTypeMask == timestampTypeMask
+
+ if m.Version == 1 {
+ if err := (Timestamp{&m.Timestamp}).decode(pd); err != nil {
+ return err
+ }
+ }
+
+ m.Key, err = pd.getBytes()
+ if err != nil {
+ return err
+ }
+
+ m.Value, err = pd.getBytes()
+ if err != nil {
+ return err
+ }
+
+ // Required for deep equal assertion during tests but might be useful
+ // for future metrics about the compression ratio in fetch requests
+ m.compressedSize = len(m.Value)
+
+ if m.Value != nil && m.Codec != CompressionNone {
+ m.Value, err = decompress(m.Codec, m.Value)
+ if err != nil {
+ return err
+ }
+
+ if err := m.decodeSet(); err != nil {
+ return err
+ }
+ }
+
+ return pd.pop()
+}
+
+// decodes a message set from a previously encoded bulk-message
+func (m *Message) decodeSet() (err error) {
+ pd := realDecoder{raw: m.Value}
+ m.Set = &MessageSet{}
+ return m.Set.decode(&pd)
+}
diff --git a/vendor/github.com/IBM/sarama/message_set.go b/vendor/github.com/IBM/sarama/message_set.go
new file mode 100644
index 0000000..5bc2112
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/message_set.go
@@ -0,0 +1,112 @@
+package sarama
+
+import "errors"
+
+type MessageBlock struct {
+ Offset int64
+ Msg *Message
+}
+
+// Messages convenience helper which returns either all the
+// messages that are wrapped in this block
+func (msb *MessageBlock) Messages() []*MessageBlock {
+ if msb.Msg.Set != nil {
+ return msb.Msg.Set.Messages
+ }
+ return []*MessageBlock{msb}
+}
+
+func (msb *MessageBlock) encode(pe packetEncoder) error {
+ pe.putInt64(msb.Offset)
+ pe.push(&lengthField{})
+ err := msb.Msg.encode(pe)
+ if err != nil {
+ return err
+ }
+ return pe.pop()
+}
+
+func (msb *MessageBlock) decode(pd packetDecoder) (err error) {
+ if msb.Offset, err = pd.getInt64(); err != nil {
+ return err
+ }
+
+ lengthDecoder := acquireLengthField()
+ defer releaseLengthField(lengthDecoder)
+
+ if err = pd.push(lengthDecoder); err != nil {
+ return err
+ }
+
+ msb.Msg = new(Message)
+ if err = msb.Msg.decode(pd); err != nil {
+ return err
+ }
+
+ if err = pd.pop(); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+type MessageSet struct {
+ PartialTrailingMessage bool // whether the set on the wire contained an incomplete trailing MessageBlock
+ OverflowMessage bool // whether the set on the wire contained an overflow message
+ Messages []*MessageBlock
+}
+
+func (ms *MessageSet) encode(pe packetEncoder) error {
+ for i := range ms.Messages {
+ err := ms.Messages[i].encode(pe)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (ms *MessageSet) decode(pd packetDecoder) (err error) {
+ ms.Messages = nil
+
+ for pd.remaining() > 0 {
+ magic, err := magicValue(pd)
+ if err != nil {
+ if errors.Is(err, ErrInsufficientData) {
+ ms.PartialTrailingMessage = true
+ return nil
+ }
+ return err
+ }
+
+ if magic > 1 {
+ return nil
+ }
+
+ msb := new(MessageBlock)
+ err = msb.decode(pd)
+ if err == nil {
+ ms.Messages = append(ms.Messages, msb)
+ } else if errors.Is(err, ErrInsufficientData) {
+ // As an optimization the server is allowed to return a partial message at the
+ // end of the message set. Clients should handle this case. So we just ignore such things.
+ if msb.Offset == -1 {
+ // This is an overflow message caused by chunked down conversion
+ ms.OverflowMessage = true
+ } else {
+ ms.PartialTrailingMessage = true
+ }
+ return nil
+ } else {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (ms *MessageSet) addMessage(msg *Message) {
+ block := new(MessageBlock)
+ block.Msg = msg
+ ms.Messages = append(ms.Messages, block)
+}
diff --git a/vendor/github.com/IBM/sarama/metadata.go b/vendor/github.com/IBM/sarama/metadata.go
new file mode 100644
index 0000000..20620b3
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/metadata.go
@@ -0,0 +1,239 @@
+package sarama
+
+import (
+ "sync"
+)
+
+type metadataRefresh func(topics []string) error
+
+// currentRefresh makes sure sarama does not issue metadata requests
+// in parallel. If we need to refresh the metadata for a list of topics,
+// this struct will check if a refresh is already ongoing, and if so, it will
+// accumulate the list of topics to refresh in the next refresh.
+// When the current refresh is over, it will queue a new metadata refresh call
+// with the accumulated list of topics.
+type currentRefresh struct {
+ // This is the function that gets called when to refresh the metadata.
+ // It is called with the list of all topics that need to be refreshed
+ // or with nil if all topics need to be refreshed.
+ refresh func(topics []string) error
+
+ mu sync.Mutex
+ ongoing bool
+ topicsMap map[string]struct{}
+ topics []string
+ allTopics bool
+ chans []chan error
+}
+
+// addTopicsFrom adds topics from the next refresh to the current refresh.
+// You need to hold the lock to call this method.
+func (r *currentRefresh) addTopicsFrom(next *nextRefresh) {
+ if next.allTopics {
+ r.allTopics = true
+ return
+ }
+ if len(next.topics) > 0 {
+ r.addTopics(next.topics)
+ }
+}
+
+// nextRefresh holds the list of topics we will need
+// to refresh in the next refresh.
+// When a refresh is ongoing, calls to RefreshMetadata() are
+// accumulated in this struct, so that we can immediately issue another
+// refresh when the current refresh is over.
+type nextRefresh struct {
+ mu sync.Mutex
+ topics []string
+ allTopics bool
+}
+
+// addTopics adds topics to the refresh.
+// You need to hold the lock to call this method.
+func (r *currentRefresh) addTopics(topics []string) {
+ if len(topics) == 0 {
+ r.allTopics = true
+ return
+ }
+ for _, topic := range topics {
+ if _, ok := r.topicsMap[topic]; ok {
+ continue
+ }
+ r.topicsMap[topic] = struct{}{}
+ r.topics = append(r.topics, topic)
+ }
+}
+
+func (r *nextRefresh) addTopics(topics []string) {
+ if len(topics) == 0 {
+ r.allTopics = true
+ // All topics are requested, so we can clear the topics
+ // that were previously accumulated.
+ r.topics = r.topics[:0]
+ return
+ }
+ r.topics = append(r.topics, topics...)
+}
+
+func (r *nextRefresh) clear() {
+ r.topics = r.topics[:0]
+ r.allTopics = false
+}
+
+func (r *currentRefresh) hasTopics(topics []string) bool {
+ if len(topics) == 0 {
+ // This means that the caller wants to know if the refresh is for all topics.
+ // In this case, we return true if the refresh is for all topics, or false if it is not.
+ return r.allTopics
+ }
+ if r.allTopics {
+ return true
+ }
+ for _, topic := range topics {
+ if _, ok := r.topicsMap[topic]; !ok {
+ return false
+ }
+ }
+ return true
+}
+
+// start starts a new refresh.
+// The refresh is started in a new goroutine, and this function
+// returns a channel on which the caller can wait for the refresh
+// to complete.
+// You need to hold the lock to call this method.
+func (r *currentRefresh) start() chan error {
+ r.ongoing = true
+ ch := r.wait()
+ topics := r.topics
+ if r.allTopics {
+ topics = nil
+ }
+ go func() {
+ err := r.refresh(topics)
+ r.mu.Lock()
+ defer r.mu.Unlock()
+
+ r.ongoing = false
+ for _, ch := range r.chans {
+ ch <- err
+ close(ch)
+ }
+ r.clear()
+ }()
+ return ch
+}
+
+// clear clears the refresh state.
+// You need to hold the lock to call this method.
+func (r *currentRefresh) clear() {
+ r.topics = r.topics[:0]
+ for key := range r.topicsMap {
+ delete(r.topicsMap, key)
+ }
+ r.allTopics = false
+ r.chans = r.chans[:0]
+}
+
+// wait returns the channel on which you can wait for the refresh
+// to complete.
+// You need to hold the lock to call this method.
+func (r *currentRefresh) wait() chan error {
+ if !r.ongoing {
+ panic("waiting for a refresh that is not ongoing")
+ }
+ ch := make(chan error, 1)
+ r.chans = append(r.chans, ch)
+ return ch
+}
+
+// singleFlightMetadataRefresher helps managing metadata refreshes.
+// It makes sure a sarama client never issues more than one metadata refresh
+// in parallel.
+type singleFlightMetadataRefresher struct {
+ current *currentRefresh
+ next *nextRefresh
+}
+
+func newSingleFlightRefresher(f func(topics []string) error) metadataRefresh {
+ return newMetadataRefresh(f).Refresh
+}
+
+func newMetadataRefresh(f func(topics []string) error) *singleFlightMetadataRefresher {
+ return &singleFlightMetadataRefresher{
+ current: ¤tRefresh{
+ topicsMap: make(map[string]struct{}),
+ refresh: f,
+ },
+ next: &nextRefresh{},
+ }
+}
+
+// Refresh is the function that clients call when they want to refresh
+// the metadata. This function blocks until a refresh is issued, and its
+// result is received, for the list of topics the caller provided.
+// If a refresh was already ongoing for this list of topics, the function
+// waits on that refresh to complete, and returns its result.
+// If a refresh was already ongoing for a different list of topics, the function
+// accumulates the list of topics to refresh in the next refresh, and queues that refresh.
+// If no refresh is ongoing, it will start a new refresh, and return its result.
+func (m *singleFlightMetadataRefresher) Refresh(topics []string) error {
+ for {
+ ch, queued := m.refreshOrQueue(topics)
+ if !queued {
+ return <-ch
+ }
+ <-ch
+ }
+}
+
+// refreshOrQueue returns a channel the refresh needs to wait on, and a boolean
+// that indicates whether waiting on the channel will return the result of that refresh
+// or whether the refresh was "queued" and the caller needs to wait for the channel to
+// return, and then call refreshOrQueue again.
+// When calling refreshOrQueue, three things can happen:
+// 1. either no refresh is ongoing.
+// In this case, a new refresh is started, and the channel that's returned will
+// contain the result of that refresh, so it returns "false" as the second return value.
+// 2. a refresh is ongoing, and it contains the topics we need.
+// In this case, the channel that's returned will contain the result of that refresh,
+// so it returns "false" as the second return value.
+// In this case, the channel that's returned will contain the result of that refresh,
+// so it returns "false" as the second return value.
+// 3. a refresh is already ongoing, but doesn't contain the topics we need. In this case,
+// the caller needs to wait for the refresh to finish, and then call refreshOrQueue again.
+// The channel that's returned is for the current refresh (not the one the caller is
+// interested in), so it returns "true" as the second return value. The caller needs to
+// wait on the channel, disregard the value, and call refreshOrQueue again.
+func (m *singleFlightMetadataRefresher) refreshOrQueue(topics []string) (chan error, bool) {
+ m.current.mu.Lock()
+ defer m.current.mu.Unlock()
+ if !m.current.ongoing {
+ // If no refresh is ongoing, we can start a new one, in which
+ // we add the topics that have been accumulated in the next refresh
+ // and the topics that have been provided by the caller.
+ m.next.mu.Lock()
+ m.current.addTopicsFrom(m.next)
+ m.next.clear()
+ m.next.mu.Unlock()
+ m.current.addTopics(topics)
+ ch := m.current.start()
+ return ch, false
+ }
+ if m.current.hasTopics(topics) {
+ // A refresh is ongoing, and we were lucky: it is refreshing the topics we need already:
+ // we just have to wait for it to finish and return its results.
+ ch := m.current.wait()
+ return ch, false
+ }
+ // There is a refresh ongoing, but it is not refreshing the topics we need.
+ // We need to wait for it to finish, and then start a new refresh.
+ ch := m.current.wait()
+ m.next.mu.Lock()
+ m.next.addTopics(topics)
+ m.next.mu.Unlock()
+ // This is where we wait for that refresh to finish, and the loop will take care
+ // of starting the new one.
+ return ch, true
+}
diff --git a/vendor/github.com/IBM/sarama/metadata_request.go b/vendor/github.com/IBM/sarama/metadata_request.go
new file mode 100644
index 0000000..f00e5ab
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/metadata_request.go
@@ -0,0 +1,211 @@
+package sarama
+
+import "encoding/base64"
+
+type Uuid [16]byte
+
+func (u Uuid) String() string {
+ return base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(u[:])
+}
+
+var NullUUID = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+
+type MetadataRequest struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // Topics contains the topics to fetch metadata for.
+ Topics []string
+ // AllowAutoTopicCreation contains a If this is true, the broker may auto-create topics that we requested which do not already exist, if it is configured to do so.
+ AllowAutoTopicCreation bool
+ IncludeClusterAuthorizedOperations bool // version 8 and up
+ IncludeTopicAuthorizedOperations bool // version 8 and up
+}
+
+func (r *MetadataRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func NewMetadataRequest(version KafkaVersion, topics []string) *MetadataRequest {
+ m := &MetadataRequest{Topics: topics}
+ if version.IsAtLeast(V2_8_0_0) {
+ m.Version = 10
+ } else if version.IsAtLeast(V2_4_0_0) {
+ m.Version = 9
+ } else if version.IsAtLeast(V2_4_0_0) {
+ m.Version = 8
+ } else if version.IsAtLeast(V2_1_0_0) {
+ m.Version = 7
+ } else if version.IsAtLeast(V2_0_0_0) {
+ m.Version = 6
+ } else if version.IsAtLeast(V1_0_0_0) {
+ m.Version = 5
+ } else if version.IsAtLeast(V0_11_0_0) {
+ m.Version = 4
+ } else if version.IsAtLeast(V0_10_1_0) {
+ m.Version = 2
+ } else if version.IsAtLeast(V0_10_0_0) {
+ m.Version = 1
+ }
+ return m
+}
+
+func (r *MetadataRequest) encode(pe packetEncoder) (err error) {
+ if r.Version < 0 || r.Version > 10 {
+ return PacketEncodingError{"invalid or unsupported MetadataRequest version field"}
+ }
+ if r.Version == 0 || len(r.Topics) > 0 {
+ if err := pe.putArrayLength(len(r.Topics)); err != nil {
+ return err
+ }
+ if r.Version <= 9 {
+ for _, topicName := range r.Topics {
+ if err := pe.putString(topicName); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+ } else { // r.Version = 10
+ for _, topicName := range r.Topics {
+ if err := pe.putRawBytes(NullUUID); err != nil {
+ return err
+ }
+ // Avoid implicit memory aliasing in for loop
+ tn := topicName
+ if err := pe.putNullableString(&tn); err != nil {
+ return err
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+ }
+ } else {
+ if err := pe.putArrayLength(-1); err != nil {
+ return err
+ }
+ }
+
+ if r.Version > 3 {
+ pe.putBool(r.AllowAutoTopicCreation)
+ }
+ if r.Version > 7 {
+ pe.putBool(r.IncludeClusterAuthorizedOperations)
+ pe.putBool(r.IncludeTopicAuthorizedOperations)
+ }
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *MetadataRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ size, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if size > 0 {
+ r.Topics = make([]string, size)
+ }
+ if version <= 9 {
+ for i := range r.Topics {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ r.Topics[i] = topic
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+ } else {
+ for i := range r.Topics {
+ if _, err = pd.getRawBytes(16); err != nil { // skip UUID
+ return err
+ }
+ topic, err := pd.getNullableString()
+ if err != nil {
+ return err
+ }
+ if topic != nil {
+ r.Topics[i] = *topic
+ }
+
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+ }
+
+ if r.Version >= 4 {
+ if r.AllowAutoTopicCreation, err = pd.getBool(); err != nil {
+ return err
+ }
+ }
+
+ if r.Version > 7 {
+ includeClusterAuthz, err := pd.getBool()
+ if err != nil {
+ return err
+ }
+ r.IncludeClusterAuthorizedOperations = includeClusterAuthz
+ includeTopicAuthz, err := pd.getBool()
+ if err != nil {
+ return err
+ }
+ r.IncludeTopicAuthorizedOperations = includeTopicAuthz
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *MetadataRequest) key() int16 {
+ return apiKeyMetadata
+}
+
+func (r *MetadataRequest) version() int16 {
+ return r.Version
+}
+
+func (r *MetadataRequest) headerVersion() int16 {
+ if r.Version >= 9 {
+ return 2
+ }
+ return 1
+}
+
+func (r *MetadataRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 10
+}
+
+func (r *MetadataRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *MetadataRequest) isFlexibleVersion(version int16) bool {
+ return version >= 9
+}
+
+func (r *MetadataRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 10:
+ return V2_8_0_0
+ case 9:
+ return V2_4_0_0
+ case 8:
+ return V2_3_0_0
+ case 7:
+ return V2_1_0_0
+ case 6:
+ return V2_0_0_0
+ case 5:
+ return V1_0_0_0
+ case 3, 4:
+ return V0_11_0_0
+ case 2:
+ return V0_10_1_0
+ case 1:
+ return V0_10_0_0
+ case 0:
+ return V0_8_2_0
+ default:
+ return V2_8_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/metadata_response.go b/vendor/github.com/IBM/sarama/metadata_response.go
new file mode 100644
index 0000000..196c075
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/metadata_response.go
@@ -0,0 +1,449 @@
+package sarama
+
+import "time"
+
+// PartitionMetadata contains each partition in the topic.
+type PartitionMetadata struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // Err contains the partition error, or 0 if there was no error.
+ Err KError
+ // ID contains the partition index.
+ ID int32
+ // Leader contains the ID of the leader broker.
+ Leader int32
+ // LeaderEpoch contains the leader epoch of this partition.
+ LeaderEpoch int32
+ // Replicas contains the set of all nodes that host this partition.
+ Replicas []int32
+ // Isr contains the set of nodes that are in sync with the leader for this partition.
+ Isr []int32
+ // OfflineReplicas contains the set of offline replicas of this partition.
+ OfflineReplicas []int32
+}
+
+func (p *PartitionMetadata) decode(pd packetDecoder, version int16) (err error) {
+ p.Version = version
+ p.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if p.ID, err = pd.getInt32(); err != nil {
+ return err
+ }
+
+ if p.Leader, err = pd.getInt32(); err != nil {
+ return err
+ }
+
+ if p.Version >= 7 {
+ if p.LeaderEpoch, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+
+ p.Replicas, err = pd.getInt32Array()
+ if err != nil {
+ return err
+ }
+
+ p.Isr, err = pd.getInt32Array()
+ if err != nil {
+ return err
+ }
+
+ if p.Version >= 5 {
+ p.OfflineReplicas, err = pd.getInt32Array()
+ if err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (p *PartitionMetadata) encode(pe packetEncoder, version int16) (err error) {
+ p.Version = version
+ pe.putKError(p.Err)
+
+ pe.putInt32(p.ID)
+
+ pe.putInt32(p.Leader)
+
+ if p.Version >= 7 {
+ pe.putInt32(p.LeaderEpoch)
+ }
+
+ err = pe.putInt32Array(p.Replicas)
+ if err != nil {
+ return err
+ }
+
+ err = pe.putInt32Array(p.Isr)
+ if err != nil {
+ return err
+ }
+
+ if p.Version >= 5 {
+ err = pe.putInt32Array(p.OfflineReplicas)
+ if err != nil {
+ return err
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+// TopicMetadata contains each topic in the response.
+type TopicMetadata struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // Err contains the topic error, or 0 if there was no error.
+ Err KError
+ // Name contains the topic name.
+ Name string
+ Uuid Uuid
+ // IsInternal contains a True if the topic is internal.
+ IsInternal bool
+ // Partitions contains each partition in the topic.
+ Partitions []*PartitionMetadata
+ TopicAuthorizedOperations int32 // Only valid for Version >= 8
+}
+
+func (t *TopicMetadata) decode(pd packetDecoder, version int16) (err error) {
+ t.Version = version
+ t.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ t.Name, err = pd.getString()
+ if err != nil {
+ return err
+ }
+
+ if t.Version >= 10 {
+ uuid, err := pd.getRawBytes(16)
+ if err != nil {
+ return err
+ }
+ t.Uuid = [16]byte{}
+ for i := 0; i < 16; i++ {
+ t.Uuid[i] = uuid[i]
+ }
+ }
+
+ if t.Version >= 1 {
+ t.IsInternal, err = pd.getBool()
+ if err != nil {
+ return err
+ }
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ t.Partitions = make([]*PartitionMetadata, n)
+ for i := 0; i < n; i++ {
+ block := &PartitionMetadata{}
+ if err := block.decode(pd, t.Version); err != nil {
+ return err
+ }
+ t.Partitions[i] = block
+ }
+
+ if t.Version >= 8 {
+ t.TopicAuthorizedOperations, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (t *TopicMetadata) encode(pe packetEncoder, version int16) (err error) {
+ t.Version = version
+ pe.putKError(t.Err)
+
+ err = pe.putString(t.Name)
+ if err != nil {
+ return err
+ }
+
+ if t.Version >= 10 {
+ err = pe.putRawBytes(t.Uuid[:])
+ if err != nil {
+ return err
+ }
+ }
+
+ if t.Version >= 1 {
+ pe.putBool(t.IsInternal)
+ }
+
+ err = pe.putArrayLength(len(t.Partitions))
+ if err != nil {
+ return err
+ }
+ for _, block := range t.Partitions {
+ if err := block.encode(pe, t.Version); err != nil {
+ return err
+ }
+ }
+
+ if t.Version >= 8 {
+ pe.putInt32(t.TopicAuthorizedOperations)
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+type MetadataResponse struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // ThrottleTimeMs contains the duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota.
+ ThrottleTimeMs int32
+ // Brokers contains each broker in the response.
+ Brokers []*Broker
+ // ClusterID contains the cluster ID that responding broker belongs to.
+ ClusterID *string
+ // ControllerID contains the ID of the controller broker.
+ ControllerID int32
+ // Topics contains each topic in the response.
+ Topics []*TopicMetadata
+ ClusterAuthorizedOperations int32 // Only valid for Version >= 8
+}
+
+func (r *MetadataResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *MetadataResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ if r.Version >= 3 {
+ if r.ThrottleTimeMs, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+
+ brokerArrayLen, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Brokers = make([]*Broker, brokerArrayLen)
+ for i := 0; i < brokerArrayLen; i++ {
+ r.Brokers[i] = new(Broker)
+ err = r.Brokers[i].decode(pd, version)
+ if err != nil {
+ return err
+ }
+ }
+
+ if r.Version >= 2 {
+ r.ClusterID, err = pd.getNullableString()
+ if err != nil {
+ return err
+ }
+ }
+
+ if r.Version >= 1 {
+ if r.ControllerID, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+
+ topicArrayLen, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Topics = make([]*TopicMetadata, topicArrayLen)
+ for i := 0; i < topicArrayLen; i++ {
+ r.Topics[i] = new(TopicMetadata)
+ err = r.Topics[i].decode(pd, version)
+ if err != nil {
+ return err
+ }
+ }
+
+ if r.Version >= 8 {
+ r.ClusterAuthorizedOperations, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *MetadataResponse) encode(pe packetEncoder) (err error) {
+ if r.Version >= 3 {
+ pe.putInt32(r.ThrottleTimeMs)
+ }
+
+ err = pe.putArrayLength(len(r.Brokers))
+ if err != nil {
+ return err
+ }
+
+ for _, broker := range r.Brokers {
+ err = broker.encode(pe, r.Version)
+ if err != nil {
+ return err
+ }
+ }
+
+ if r.Version >= 2 {
+ err = pe.putNullableString(r.ClusterID)
+ if err != nil {
+ return err
+ }
+ }
+
+ if r.Version >= 1 {
+ pe.putInt32(r.ControllerID)
+ }
+
+ err = pe.putArrayLength(len(r.Topics))
+ if err != nil {
+ return err
+ }
+ for _, block := range r.Topics {
+ if err := block.encode(pe, r.Version); err != nil {
+ return err
+ }
+ }
+
+ if r.Version >= 8 {
+ pe.putInt32(r.ClusterAuthorizedOperations)
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *MetadataResponse) key() int16 {
+ return apiKeyMetadata
+}
+
+func (r *MetadataResponse) version() int16 {
+ return r.Version
+}
+
+func (r *MetadataResponse) headerVersion() int16 {
+ if r.Version < 9 {
+ return 0
+ } else {
+ return 1
+ }
+}
+
+func (r *MetadataResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 10
+}
+
+func (r *MetadataResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *MetadataResponse) isFlexibleVersion(version int16) bool {
+ return version >= 9
+}
+
+func (r *MetadataResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 10:
+ return V2_8_0_0
+ case 9:
+ return V2_4_0_0
+ case 8:
+ return V2_3_0_0
+ case 7:
+ return V2_1_0_0
+ case 6:
+ return V2_0_0_0
+ case 5:
+ return V1_0_0_0
+ case 3, 4:
+ return V0_11_0_0
+ case 2:
+ return V0_10_1_0
+ case 1:
+ return V0_10_0_0
+ case 0:
+ return V0_8_2_0
+ default:
+ return V2_8_0_0
+ }
+}
+
+func (r *MetadataResponse) throttleTime() time.Duration {
+ return time.Duration(r.ThrottleTimeMs) * time.Millisecond
+}
+
+// testing API
+
+func (r *MetadataResponse) AddBroker(addr string, id int32) {
+ r.Brokers = append(r.Brokers, &Broker{id: id, addr: addr})
+}
+
+func (r *MetadataResponse) AddTopic(topic string, err KError) *TopicMetadata {
+ var tmatch *TopicMetadata
+
+ for _, tm := range r.Topics {
+ if tm.Name == topic {
+ tmatch = tm
+ goto foundTopic
+ }
+ }
+
+ tmatch = new(TopicMetadata)
+ tmatch.Name = topic
+ r.Topics = append(r.Topics, tmatch)
+
+foundTopic:
+
+ tmatch.Err = err
+ return tmatch
+}
+
+func (r *MetadataResponse) AddTopicPartition(topic string, partition, brokerID int32, replicas, isr []int32, offline []int32, err KError) {
+ tmatch := r.AddTopic(topic, ErrNoError)
+ var pmatch *PartitionMetadata
+
+ for _, pm := range tmatch.Partitions {
+ if pm.ID == partition {
+ pmatch = pm
+ goto foundPartition
+ }
+ }
+
+ pmatch = new(PartitionMetadata)
+ pmatch.ID = partition
+ tmatch.Partitions = append(tmatch.Partitions, pmatch)
+
+foundPartition:
+ pmatch.Leader = brokerID
+ pmatch.Replicas = replicas
+ if pmatch.Replicas == nil {
+ pmatch.Replicas = []int32{}
+ }
+ pmatch.Isr = isr
+ if pmatch.Isr == nil {
+ pmatch.Isr = []int32{}
+ }
+ pmatch.OfflineReplicas = offline
+ if pmatch.OfflineReplicas == nil {
+ pmatch.OfflineReplicas = []int32{}
+ }
+ pmatch.Err = err
+}
diff --git a/vendor/github.com/IBM/sarama/metrics.go b/vendor/github.com/IBM/sarama/metrics.go
new file mode 100644
index 0000000..4f512f2
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/metrics.go
@@ -0,0 +1,120 @@
+package sarama
+
+import (
+ "strconv"
+ "strings"
+ "sync"
+
+ "github.com/rcrowley/go-metrics"
+)
+
+// Use exponentially decaying reservoir for sampling histograms with the same defaults as the Java library:
+// 1028 elements, which offers a 99.9% confidence level with a 5% margin of error assuming a normal distribution,
+// and an alpha factor of 0.015, which heavily biases the reservoir to the past 5 minutes of measurements.
+// See https://github.com/dropwizard/metrics/blob/v3.1.0/metrics-core/src/main/java/com/codahale/metrics/ExponentiallyDecayingReservoir.java#L38
+const (
+ metricsReservoirSize = 1028
+ metricsAlphaFactor = 0.015
+)
+
+func getOrRegisterHistogram(name string, r metrics.Registry) metrics.Histogram {
+ return r.GetOrRegister(name, func() metrics.Histogram {
+ return metrics.NewHistogram(metrics.NewExpDecaySample(metricsReservoirSize, metricsAlphaFactor))
+ }).(metrics.Histogram)
+}
+
+func getMetricNameForBroker(name string, broker *Broker) string {
+ // Use broker id like the Java client as it does not contain '.' or ':' characters that
+ // can be interpreted as special character by monitoring tool (e.g. Graphite)
+ return name + "-for-broker-" + strconv.FormatInt(int64(broker.ID()), 10)
+}
+
+func getMetricNameForTopic(name string, topic string) string {
+ // Convert dot to _ since reporters like Graphite typically use dot to represent hierarchy
+ // cf. KAFKA-1902 and KAFKA-2337
+ return name + "-for-topic-" + strings.ReplaceAll(topic, ".", "_")
+}
+
+func getOrRegisterTopicMeter(name string, topic string, r metrics.Registry) metrics.Meter {
+ return metrics.GetOrRegisterMeter(getMetricNameForTopic(name, topic), r)
+}
+
+func getOrRegisterTopicHistogram(name string, topic string, r metrics.Registry) metrics.Histogram {
+ return getOrRegisterHistogram(getMetricNameForTopic(name, topic), r)
+}
+
+// cleanupRegistry is an implementation of metrics.Registry that allows
+// to unregister from the parent registry only those metrics
+// that have been registered in cleanupRegistry
+type cleanupRegistry struct {
+ parent metrics.Registry
+ metrics map[string]struct{}
+ mutex sync.RWMutex
+}
+
+func newCleanupRegistry(parent metrics.Registry) metrics.Registry {
+ return &cleanupRegistry{
+ parent: parent,
+ metrics: map[string]struct{}{},
+ }
+}
+
+func (r *cleanupRegistry) Each(fn func(string, interface{})) {
+ r.mutex.RLock()
+ defer r.mutex.RUnlock()
+ wrappedFn := func(name string, iface interface{}) {
+ if _, ok := r.metrics[name]; ok {
+ fn(name, iface)
+ }
+ }
+ r.parent.Each(wrappedFn)
+}
+
+func (r *cleanupRegistry) Get(name string) interface{} {
+ r.mutex.RLock()
+ defer r.mutex.RUnlock()
+ if _, ok := r.metrics[name]; ok {
+ return r.parent.Get(name)
+ }
+ return nil
+}
+
+func (r *cleanupRegistry) GetOrRegister(name string, metric interface{}) interface{} {
+ r.mutex.Lock()
+ defer r.mutex.Unlock()
+ r.metrics[name] = struct{}{}
+ return r.parent.GetOrRegister(name, metric)
+}
+
+func (r *cleanupRegistry) Register(name string, metric interface{}) error {
+ r.mutex.Lock()
+ defer r.mutex.Unlock()
+ r.metrics[name] = struct{}{}
+ return r.parent.Register(name, metric)
+}
+
+func (r *cleanupRegistry) RunHealthchecks() {
+ r.parent.RunHealthchecks()
+}
+
+func (r *cleanupRegistry) GetAll() map[string]map[string]interface{} {
+ return r.parent.GetAll()
+}
+
+func (r *cleanupRegistry) Unregister(name string) {
+ r.mutex.Lock()
+ defer r.mutex.Unlock()
+ if _, ok := r.metrics[name]; ok {
+ delete(r.metrics, name)
+ r.parent.Unregister(name)
+ }
+}
+
+func (r *cleanupRegistry) UnregisterAll() {
+ r.mutex.Lock()
+ defer r.mutex.Unlock()
+ for name := range r.metrics {
+ delete(r.metrics, name)
+ r.parent.Unregister(name)
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/mockbroker.go b/vendor/github.com/IBM/sarama/mockbroker.go
new file mode 100644
index 0000000..d913d44
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/mockbroker.go
@@ -0,0 +1,467 @@
+package sarama
+
+import (
+ "bytes"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "io"
+ "maps"
+ "net"
+ "reflect"
+ "strconv"
+ "sync"
+ "syscall"
+ "time"
+
+ "github.com/davecgh/go-spew/spew"
+)
+
+const (
+ expectationTimeout = 500 * time.Millisecond
+)
+
+type GSSApiHandlerFunc func([]byte) []byte
+
+type requestHandlerFunc func(req *request) (res encoderWithHeader)
+
+// RequestNotifierFunc is invoked when a mock broker processes a request successfully
+// and will provides the number of bytes read and written.
+type RequestNotifierFunc func(bytesRead, bytesWritten int)
+
+// MockBroker is a mock Kafka broker that is used in unit tests. It is exposed
+// to facilitate testing of higher level or specialized consumers and producers
+// built on top of Sarama. Note that it does not 'mimic' the Kafka API protocol,
+// but rather provides a facility to do that. It takes care of the TCP
+// transport, request unmarshalling, response marshaling, and makes it the test
+// writer responsibility to program correct according to the Kafka API protocol
+// MockBroker behavior.
+//
+// MockBroker is implemented as a TCP server listening on a kernel-selected
+// localhost port that can accept many connections. It reads Kafka requests
+// from that connection and returns responses programmed by the SetHandlerByMap
+// function. If a MockBroker receives a request that it has no programmed
+// response for, then it returns nothing and the request times out.
+//
+// A set of MockRequest builders to define mappings used by MockBroker is
+// provided by Sarama. But users can develop MockRequests of their own and use
+// them along with or instead of the standard ones.
+//
+// When running tests with MockBroker it is strongly recommended to specify
+// a timeout to `go test` so that if the broker hangs waiting for a response,
+// the test panics.
+//
+// It is not necessary to prefix message length or correlation ID to your
+// response bytes, the server does that automatically as a convenience.
+type MockBroker struct {
+ brokerID int32
+ port int32
+ closing chan none
+ stopper chan none
+ expectations chan encoderWithHeader
+ listener net.Listener
+ t TestReporter
+ latency time.Duration
+ handler requestHandlerFunc
+ notifier RequestNotifierFunc
+ history []RequestResponse
+ lock sync.Mutex
+ gssApiHandler GSSApiHandlerFunc
+}
+
+// RequestResponse represents a Request/Response pair processed by MockBroker.
+type RequestResponse struct {
+ Request protocolBody
+ Response encoder
+}
+
+// SetLatency makes broker pause for the specified period every time before
+// replying.
+func (b *MockBroker) SetLatency(latency time.Duration) {
+ b.latency = latency
+}
+
+// SetHandlerByMap defines mapping of Request types to MockResponses. When a
+// request is received by the broker, it looks up the request type in the map
+// and uses the found MockResponse instance to generate an appropriate reply.
+// If the request type is not found in the map then nothing is sent.
+func (b *MockBroker) SetHandlerByMap(handlerMap map[string]MockResponse) {
+ fnMap := maps.Clone(handlerMap)
+ b.setHandler(func(req *request) (res encoderWithHeader) {
+ reqTypeName := reflect.TypeOf(req.body).Elem().Name()
+ mockResponse := fnMap[reqTypeName]
+ if mockResponse == nil {
+ return nil
+ }
+ return mockResponse.For(req.body)
+ })
+}
+
+// SetHandlerFuncByMap defines mapping of Request types to RequestHandlerFunc. When a
+// request is received by the broker, it looks up the request type in the map
+// and invoke the found RequestHandlerFunc instance to generate an appropriate reply.
+func (b *MockBroker) SetHandlerFuncByMap(handlerMap map[string]requestHandlerFunc) {
+ fnMap := maps.Clone(handlerMap)
+ b.setHandler(func(req *request) (res encoderWithHeader) {
+ reqTypeName := reflect.TypeOf(req.body).Elem().Name()
+ return fnMap[reqTypeName](req)
+ })
+}
+
+// SetNotifier set a function that will get invoked whenever a request has been
+// processed successfully and will provide the number of bytes read and written
+func (b *MockBroker) SetNotifier(notifier RequestNotifierFunc) {
+ b.lock.Lock()
+ b.notifier = notifier
+ b.lock.Unlock()
+}
+
+// BrokerID returns broker ID assigned to the broker.
+func (b *MockBroker) BrokerID() int32 {
+ return b.brokerID
+}
+
+// History returns a slice of RequestResponse pairs in the order they were
+// processed by the broker. Note that in case of multiple connections to the
+// broker the order expected by a test can be different from the order recorded
+// in the history, unless some synchronization is implemented in the test.
+func (b *MockBroker) History() []RequestResponse {
+ b.lock.Lock()
+ history := make([]RequestResponse, len(b.history))
+ copy(history, b.history)
+ b.lock.Unlock()
+ return history
+}
+
+// Port returns the TCP port number the broker is listening for requests on.
+func (b *MockBroker) Port() int32 {
+ return b.port
+}
+
+// Addr returns the broker connection string in the form "<address>:<port>".
+func (b *MockBroker) Addr() string {
+ return b.listener.Addr().String()
+}
+
+// Close terminates the broker blocking until it stops internal goroutines and
+// releases all resources.
+func (b *MockBroker) Close() {
+ close(b.expectations)
+ if len(b.expectations) > 0 {
+ buf := bytes.NewBufferString(fmt.Sprintf("mockbroker/%d: not all expectations were satisfied! Still waiting on:\n", b.BrokerID()))
+ for e := range b.expectations {
+ _, _ = buf.WriteString(spew.Sdump(e))
+ }
+ b.t.Error(buf.String())
+ }
+ close(b.closing)
+ <-b.stopper
+}
+
+// setHandler sets the specified function as the request handler. Whenever
+// a mock broker reads a request from the wire it passes the request to the
+// function and sends back whatever the handler function returns.
+func (b *MockBroker) setHandler(handler requestHandlerFunc) {
+ b.lock.Lock()
+ b.handler = handler
+ b.lock.Unlock()
+}
+
+func (b *MockBroker) serverLoop() {
+ defer close(b.stopper)
+ var err error
+ var conn net.Conn
+
+ go func() {
+ <-b.closing
+ err := b.listener.Close()
+ if err != nil {
+ b.t.Error(err)
+ }
+ }()
+
+ wg := &sync.WaitGroup{}
+ i := 0
+ for conn, err = b.listener.Accept(); err == nil; conn, err = b.listener.Accept() {
+ wg.Add(1)
+ go b.handleRequests(conn, i, wg)
+ i++
+ }
+ wg.Wait()
+ if !isConnectionClosedError(err) {
+ Logger.Printf("*** mockbroker/%d: listener closed, err=%v", b.BrokerID(), err)
+ }
+}
+
+func (b *MockBroker) SetGSSAPIHandler(handler GSSApiHandlerFunc) {
+ b.gssApiHandler = handler
+}
+
+func (b *MockBroker) readToBytes(r io.Reader) ([]byte, error) {
+ var (
+ bytesRead int
+ lengthBytes = make([]byte, 4)
+ )
+
+ if _, err := io.ReadFull(r, lengthBytes); err != nil {
+ return nil, err
+ }
+
+ bytesRead += len(lengthBytes)
+ length := int32(binary.BigEndian.Uint32(lengthBytes))
+
+ if length <= 4 || length > MaxRequestSize {
+ return nil, PacketDecodingError{fmt.Sprintf("message of length %d too large or too small", length)}
+ }
+
+ encodedReq := make([]byte, length)
+ if _, err := io.ReadFull(r, encodedReq); err != nil {
+ return nil, err
+ }
+
+ bytesRead += len(encodedReq)
+
+ fullBytes := append(lengthBytes, encodedReq...)
+
+ return fullBytes, nil
+}
+
+func (b *MockBroker) isGSSAPI(buffer []byte) bool {
+ return buffer[4] == 0x60 || bytes.Equal(buffer[4:6], []byte{0x05, 0x04})
+}
+
+func (b *MockBroker) handleRequests(conn io.ReadWriteCloser, idx int, wg *sync.WaitGroup) {
+ defer wg.Done()
+ defer func() {
+ _ = conn.Close()
+ }()
+ s := spew.NewDefaultConfig()
+ s.MaxDepth = 1
+ Logger.Printf("*** mockbroker/%d/%d: connection opened", b.BrokerID(), idx)
+ var err error
+
+ abort := make(chan none)
+ defer close(abort)
+ go func() {
+ select {
+ case <-b.closing:
+ _ = conn.Close()
+ case <-abort:
+ }
+ }()
+
+ var bytesWritten int
+ var bytesRead int
+ for {
+ buffer, err := b.readToBytes(conn)
+ if err != nil {
+ if !isConnectionClosedError(err) {
+ Logger.Printf("*** mockbroker/%d/%d: invalid request: err=%+v, %+v", b.brokerID, idx, err, spew.Sdump(buffer))
+ b.serverError(err)
+ }
+ break
+ }
+
+ bytesWritten = 0
+ if !b.isGSSAPI(buffer) {
+ req, br, err := decodeRequest(bytes.NewReader(buffer))
+ bytesRead = br
+ if err != nil {
+ if !isConnectionClosedError(err) {
+ Logger.Printf("*** mockbroker/%d/%d: invalid request: err=%+v, %+v", b.brokerID, idx, err, spew.Sdump(req))
+ b.serverError(err)
+ }
+ break
+ }
+
+ if b.latency > 0 {
+ time.Sleep(b.latency)
+ }
+
+ b.lock.Lock()
+ res := b.handler(req)
+ b.history = append(b.history, RequestResponse{req.body, res})
+ b.lock.Unlock()
+
+ if res == nil {
+ Logger.Printf("*** mockbroker/%d/%d: ignored %v", b.brokerID, idx, spew.Sdump(req))
+ continue
+ }
+ Logger.Printf(
+ "*** mockbroker/%d/%d: replied to %T with %T\n-> %s\n-> %s",
+ b.brokerID, idx, req.body, res,
+ s.Sprintf("%#v", req.body),
+ s.Sprintf("%#v", res),
+ )
+
+ encodedRes, err := encode(res, nil)
+ if err != nil {
+ b.serverError(fmt.Errorf("failed to encode %T - %w", res, err))
+ break
+ }
+ if len(encodedRes) == 0 {
+ b.lock.Lock()
+ if b.notifier != nil {
+ b.notifier(bytesRead, 0)
+ }
+ b.lock.Unlock()
+ continue
+ }
+
+ resHeader := b.encodeHeader(res.headerVersion(), req.correlationID, uint32(len(encodedRes)))
+ if _, err = conn.Write(resHeader); err != nil {
+ b.serverError(err)
+ break
+ }
+ if _, err = conn.Write(encodedRes); err != nil {
+ b.serverError(err)
+ break
+ }
+ bytesWritten = len(resHeader) + len(encodedRes)
+ } else {
+ // GSSAPI is not part of kafka protocol, but is supported for authentication proposes.
+ // Don't support history for this kind of request as is only used for test GSSAPI authentication mechanism
+ b.lock.Lock()
+ res := b.gssApiHandler(buffer)
+ b.lock.Unlock()
+ if res == nil {
+ Logger.Printf("*** mockbroker/%d/%d: ignored %v", b.brokerID, idx, spew.Sdump(buffer))
+ continue
+ }
+ if _, err = conn.Write(res); err != nil {
+ b.serverError(err)
+ break
+ }
+ bytesWritten = len(res)
+ }
+
+ b.lock.Lock()
+ if b.notifier != nil {
+ b.notifier(bytesRead, bytesWritten)
+ }
+ b.lock.Unlock()
+ }
+ Logger.Printf("*** mockbroker/%d/%d: connection closed, err=%v", b.BrokerID(), idx, err)
+}
+
+func (b *MockBroker) encodeHeader(headerVersion int16, correlationId int32, payloadLength uint32) []byte {
+ headerLength := uint32(8)
+
+ if headerVersion >= 1 {
+ headerLength = 9
+ }
+
+ resHeader := make([]byte, headerLength)
+ binary.BigEndian.PutUint32(resHeader, payloadLength+headerLength-4)
+ binary.BigEndian.PutUint32(resHeader[4:], uint32(correlationId))
+
+ if headerVersion >= 1 {
+ binary.PutUvarint(resHeader[8:], 0)
+ }
+
+ return resHeader
+}
+
+func (b *MockBroker) defaultRequestHandler(req *request) (res encoderWithHeader) {
+ select {
+ case res, ok := <-b.expectations:
+ if !ok {
+ return nil
+ }
+ return res
+ case <-time.After(expectationTimeout):
+ return nil
+ }
+}
+
+func isConnectionClosedError(err error) bool {
+ var result bool
+ opError := &net.OpError{}
+ if errors.As(err, &opError) {
+ result = true
+ } else if errors.Is(err, io.EOF) {
+ result = true
+ } else if err.Error() == "use of closed network connection" {
+ result = true
+ }
+
+ return result
+}
+
+func (b *MockBroker) serverError(err error) {
+ b.t.Helper()
+ if isConnectionClosedError(err) {
+ return
+ }
+ b.t.Errorf(err.Error())
+}
+
+// NewMockBroker launches a fake Kafka broker. It takes a TestReporter as provided by the
+// test framework and a channel of responses to use. If an error occurs it is
+// simply logged to the TestReporter and the broker exits.
+func NewMockBroker(t TestReporter, brokerID int32) *MockBroker {
+ return NewMockBrokerAddr(t, brokerID, "localhost:0")
+}
+
+// NewMockBrokerAddr behaves like newMockBroker but listens on the address you give
+// it rather than just some ephemeral port.
+func NewMockBrokerAddr(t TestReporter, brokerID int32, addr string) *MockBroker {
+ var (
+ listener net.Listener
+ err error
+ )
+
+ // retry up to 20 times if address already in use (e.g., if replacing broker which hasn't cleanly shutdown)
+ for i := 0; i < 20; i++ {
+ listener, err = net.Listen("tcp", addr)
+ if err != nil {
+ if errors.Is(err, syscall.EADDRINUSE) {
+ Logger.Printf("*** mockbroker/%d waiting for %s (address already in use)", brokerID, addr)
+ time.Sleep(time.Millisecond * 100)
+ continue
+ }
+ t.Fatal(err)
+ }
+ break
+ }
+
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return NewMockBrokerListener(t, brokerID, listener)
+}
+
+// NewMockBrokerListener behaves like newMockBrokerAddr but accepts connections on the listener specified.
+func NewMockBrokerListener(t TestReporter, brokerID int32, listener net.Listener) *MockBroker {
+ var err error
+
+ broker := &MockBroker{
+ closing: make(chan none),
+ stopper: make(chan none),
+ t: t,
+ brokerID: brokerID,
+ expectations: make(chan encoderWithHeader, 512),
+ listener: listener,
+ }
+ broker.handler = broker.defaultRequestHandler
+
+ Logger.Printf("*** mockbroker/%d listening on %s\n", brokerID, broker.listener.Addr().String())
+ _, portStr, err := net.SplitHostPort(broker.listener.Addr().String())
+ if err != nil {
+ t.Fatal(err)
+ }
+ tmp, err := strconv.ParseInt(portStr, 10, 32)
+ if err != nil {
+ t.Fatal(err)
+ }
+ broker.port = int32(tmp)
+
+ go broker.serverLoop()
+
+ return broker
+}
+
+func (b *MockBroker) Returns(e encoderWithHeader) {
+ b.expectations <- e
+}
diff --git a/vendor/github.com/IBM/sarama/mockkerberos.go b/vendor/github.com/IBM/sarama/mockkerberos.go
new file mode 100644
index 0000000..a43607e
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/mockkerberos.go
@@ -0,0 +1,125 @@
+package sarama
+
+import (
+ "encoding/binary"
+ "encoding/hex"
+
+ "github.com/jcmturner/gokrb5/v8/credentials"
+ "github.com/jcmturner/gokrb5/v8/gssapi"
+ "github.com/jcmturner/gokrb5/v8/iana/keyusage"
+ "github.com/jcmturner/gokrb5/v8/messages"
+ "github.com/jcmturner/gokrb5/v8/types"
+)
+
+type KafkaGSSAPIHandler struct {
+ client *MockKerberosClient
+ badResponse bool
+ badKeyChecksum bool
+}
+
+func (h *KafkaGSSAPIHandler) MockKafkaGSSAPI(buffer []byte) []byte {
+ // Default payload used for verify
+ err := h.client.Login() // Mock client construct keys when login
+ if err != nil {
+ return nil
+ }
+ if h.badResponse { // Returns trash
+ return []byte{0x00, 0x00, 0x00, 0x01, 0xAD}
+ }
+
+ pack := gssapi.WrapToken{
+ Flags: KRB5_USER_AUTH,
+ EC: 12,
+ RRC: 0,
+ SndSeqNum: 3398292281,
+ Payload: []byte{0x11, 0x00}, // 1100
+ }
+ // Compute checksum
+ if h.badKeyChecksum {
+ pack.CheckSum = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}
+ } else {
+ err = pack.SetCheckSum(h.client.ASRep.DecryptedEncPart.Key, keyusage.GSSAPI_ACCEPTOR_SEAL)
+ if err != nil {
+ return nil
+ }
+ }
+
+ packBytes, err := pack.Marshal()
+ if err != nil {
+ return nil
+ }
+ lenBytes := len(packBytes)
+ response := make([]byte, lenBytes+4)
+ copy(response[4:], packBytes)
+ binary.BigEndian.PutUint32(response, uint32(lenBytes))
+ return response
+}
+
+type MockKerberosClient struct {
+ asRepBytes string
+ ASRep messages.ASRep
+ credentials *credentials.Credentials
+ mockError error
+ errorStage string
+}
+
+func (c *MockKerberosClient) Login() error {
+ if c.errorStage == "login" && c.mockError != nil {
+ return c.mockError
+ }
+ c.asRepBytes = "6b8202e9308202e5a003020105a10302010ba22b30293027a103020113a220041e301c301aa003020112a1131b114" +
+ "558414d504c452e434f4d636c69656e74a30d1b0b4558414d504c452e434f4da4133011a003020101a10a30081b06636c69656e7" +
+ "4a5820156618201523082014ea003020105a10d1b0b4558414d504c452e434f4da220301ea003020102a11730151b066b7262746" +
+ "7741b0b4558414d504c452e434f4da382011430820110a003020112a103020101a28201020481ffdb9891175d106818e61008c51" +
+ "d0b3462bca92f3bf9d4cfa82de4c4d7aff9994ec87c573e3a3d54dcb2bb79618c76f2bf4a3d006f90d5bdbd049bc18f48be39203" +
+ "549ca02acaf63f292b12404f9b74c34b83687119d8f56552ccc0c50ebee2a53bb114c1b4619bb1d5d31f0f49b4d40a08a9b4c046" +
+ "2e1398d0b648be1c0e50c552ad16e1d8d8e74263dd0bf0ec591e4797dfd40a9a1be4ae830d03a306e053fd7586fef84ffc5e4a83" +
+ "7c3122bf3e6a40fe87e84019f6283634461b955712b44a5f7386c278bff94ec2c2dc0403247e29c2450e853471ceababf9b8911f" +
+ "997f2e3010b046d2c49eb438afb0f4c210821e80d4ffa4c9521eb895dcd68610b3feaa682012c30820128a003020112a282011f0" +
+ "482011bce73cbce3f1dd17661c412005f0f2257c756fe8e98ff97e6ec24b7bab66e5fd3a3827aeeae4757af0c6e892948122d8b2" +
+ "03c8df48df0ef5d142d0e416d688f11daa0fcd63d96bdd431d02b8e951c664eeff286a2be62383d274a04016d5f0e141da58cb86" +
+ "331de64063062f4f885e8e9ce5b181ca2fdc67897c5995e0ae1ae0c171a64493ff7bd91bc6d89cd4fce1e2b3ea0a10e34b0d5eda" +
+ "aa38ee727b50c5632ed1d2f2b457908e616178d0d80b72af209fb8ac9dbaa1768fa45931392b36b6d8c12400f8ded2efaa0654d0" +
+ "da1db966e8b5aab4706c800f95d559664646041fdb38b411c62fc0fbe0d25083a28562b0e1c8df16e62e9d5626b0addee489835f" +
+ "eedb0f26c05baa596b69b17f47920aa64b29dc77cfcc97ba47885"
+ apRepBytes, err := hex.DecodeString(c.asRepBytes)
+ if err != nil {
+ return err
+ }
+ err = c.ASRep.Unmarshal(apRepBytes)
+ if err != nil {
+ return err
+ }
+ c.credentials = credentials.New("client", "EXAMPLE.COM").WithPassword("qwerty")
+ _, err = c.ASRep.DecryptEncPart(c.credentials)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (c *MockKerberosClient) GetServiceTicket(spn string) (messages.Ticket, types.EncryptionKey, error) {
+ if c.errorStage == "service_ticket" && c.mockError != nil {
+ return messages.Ticket{}, types.EncryptionKey{}, c.mockError
+ }
+ return c.ASRep.Ticket, c.ASRep.DecryptedEncPart.Key, nil
+}
+
+func (c *MockKerberosClient) Domain() string {
+ return "EXAMPLE.COM"
+}
+
+func (c *MockKerberosClient) CName() types.PrincipalName {
+ p := types.PrincipalName{
+ NameType: KRB5_USER_AUTH,
+ NameString: []string{
+ "kafka",
+ "kafka",
+ },
+ }
+ return p
+}
+
+func (c *MockKerberosClient) Destroy() {
+ // Do nothing.
+}
diff --git a/vendor/github.com/IBM/sarama/mockresponses.go b/vendor/github.com/IBM/sarama/mockresponses.go
new file mode 100644
index 0000000..2c35279
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/mockresponses.go
@@ -0,0 +1,1531 @@
+package sarama
+
+import (
+ "fmt"
+ "strings"
+ "sync"
+)
+
+// TestReporter has methods matching go's testing.T to avoid importing
+// `testing` in the main part of the library.
+type TestReporter interface {
+ Error(...interface{})
+ Errorf(string, ...interface{})
+ Fatal(...interface{})
+ Fatalf(string, ...interface{})
+ Helper()
+}
+
+// MockResponse is a response builder interface it defines one method that
+// allows generating a response based on a request body. MockResponses are used
+// to program behavior of MockBroker in tests.
+type MockResponse interface {
+ For(reqBody versionedDecoder) (res encoderWithHeader)
+}
+
+// MockWrapper is a mock response builder that returns a particular concrete
+// response regardless of the actual request passed to the `For` method.
+type MockWrapper struct {
+ res encoderWithHeader
+}
+
+func (mw *MockWrapper) For(reqBody versionedDecoder) (res encoderWithHeader) {
+ return mw.res
+}
+
+func NewMockWrapper(res encoderWithHeader) *MockWrapper {
+ return &MockWrapper{res: res}
+}
+
+// MockSequence is a mock response builder that is created from a sequence of
+// concrete responses. Every time when a `MockBroker` calls its `For` method
+// the next response from the sequence is returned. When the end of the
+// sequence is reached the last element from the sequence is returned.
+type MockSequence struct {
+ responses []MockResponse
+}
+
+func NewMockSequence(responses ...interface{}) *MockSequence {
+ ms := &MockSequence{}
+ ms.responses = make([]MockResponse, len(responses))
+ for i, res := range responses {
+ switch res := res.(type) {
+ case MockResponse:
+ ms.responses[i] = res
+ case encoderWithHeader:
+ ms.responses[i] = NewMockWrapper(res)
+ default:
+ panic(fmt.Sprintf("Unexpected response type: %T", res))
+ }
+ }
+ return ms
+}
+
+func (mc *MockSequence) For(reqBody versionedDecoder) (res encoderWithHeader) {
+ res = mc.responses[0].For(reqBody)
+ if len(mc.responses) > 1 {
+ mc.responses = mc.responses[1:]
+ }
+ return res
+}
+
+type MockListGroupsResponse struct {
+ groups map[string]string
+ t TestReporter
+}
+
+func NewMockListGroupsResponse(t TestReporter) *MockListGroupsResponse {
+ return &MockListGroupsResponse{
+ groups: make(map[string]string),
+ t: t,
+ }
+}
+
+func (m *MockListGroupsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ request := reqBody.(*ListGroupsRequest)
+ response := &ListGroupsResponse{
+ Version: request.Version,
+ Groups: m.groups,
+ }
+ return response
+}
+
+func (m *MockListGroupsResponse) AddGroup(groupID, protocolType string) *MockListGroupsResponse {
+ m.groups[groupID] = protocolType
+ return m
+}
+
+type MockDescribeGroupsResponse struct {
+ groups map[string]*GroupDescription
+ t TestReporter
+}
+
+func NewMockDescribeGroupsResponse(t TestReporter) *MockDescribeGroupsResponse {
+ return &MockDescribeGroupsResponse{
+ t: t,
+ groups: make(map[string]*GroupDescription),
+ }
+}
+
+func (m *MockDescribeGroupsResponse) AddGroupDescription(groupID string, description *GroupDescription) *MockDescribeGroupsResponse {
+ m.groups[groupID] = description
+ return m
+}
+
+func (m *MockDescribeGroupsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ request := reqBody.(*DescribeGroupsRequest)
+
+ response := &DescribeGroupsResponse{Version: request.version()}
+ for _, requestedGroup := range request.Groups {
+ if group, ok := m.groups[requestedGroup]; ok {
+ response.Groups = append(response.Groups, group)
+ } else {
+ // Mimic real kafka - if a group doesn't exist, return
+ // an entry with state "Dead"
+ response.Groups = append(response.Groups, &GroupDescription{
+ GroupId: requestedGroup,
+ State: "Dead",
+ })
+ }
+ }
+
+ return response
+}
+
+// MockMetadataResponse is a `MetadataResponse` builder.
+type MockMetadataResponse struct {
+ controllerID int32
+ errors map[string]KError
+ leaders map[string]map[int32]int32
+ brokers map[string]int32
+ t TestReporter
+}
+
+func NewMockMetadataResponse(t TestReporter) *MockMetadataResponse {
+ return &MockMetadataResponse{
+ errors: make(map[string]KError),
+ leaders: make(map[string]map[int32]int32),
+ brokers: make(map[string]int32),
+ t: t,
+ }
+}
+
+func (mmr *MockMetadataResponse) SetError(topic string, kerror KError) *MockMetadataResponse {
+ mmr.errors[topic] = kerror
+ return mmr
+}
+
+func (mmr *MockMetadataResponse) SetLeader(topic string, partition, brokerID int32) *MockMetadataResponse {
+ partitions := mmr.leaders[topic]
+ if partitions == nil {
+ partitions = make(map[int32]int32)
+ mmr.leaders[topic] = partitions
+ }
+ partitions[partition] = brokerID
+ return mmr
+}
+
+func (mmr *MockMetadataResponse) SetBroker(addr string, brokerID int32) *MockMetadataResponse {
+ mmr.brokers[addr] = brokerID
+ return mmr
+}
+
+func (mmr *MockMetadataResponse) SetController(brokerID int32) *MockMetadataResponse {
+ mmr.controllerID = brokerID
+ return mmr
+}
+
+func (mmr *MockMetadataResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ metadataRequest := reqBody.(*MetadataRequest)
+ metadataResponse := &MetadataResponse{
+ Version: metadataRequest.version(),
+ ControllerID: mmr.controllerID,
+ }
+ for addr, brokerID := range mmr.brokers {
+ metadataResponse.AddBroker(addr, brokerID)
+ }
+
+ // Generate set of replicas
+ var replicas []int32
+ var offlineReplicas []int32
+ for _, brokerID := range mmr.brokers {
+ replicas = append(replicas, brokerID)
+ }
+
+ if len(metadataRequest.Topics) == 0 {
+ for topic, partitions := range mmr.leaders {
+ for partition, brokerID := range partitions {
+ metadataResponse.AddTopicPartition(topic, partition, brokerID, replicas, replicas, offlineReplicas, ErrNoError)
+ }
+ }
+ for topic, err := range mmr.errors {
+ metadataResponse.AddTopic(topic, err)
+ }
+ return metadataResponse
+ }
+ for _, topic := range metadataRequest.Topics {
+ leaders, ok := mmr.leaders[topic]
+ if !ok {
+ if err, ok := mmr.errors[topic]; ok {
+ metadataResponse.AddTopic(topic, err)
+ } else {
+ metadataResponse.AddTopic(topic, ErrUnknownTopicOrPartition)
+ }
+ continue
+ }
+ for partition, brokerID := range leaders {
+ metadataResponse.AddTopicPartition(topic, partition, brokerID, replicas, replicas, offlineReplicas, ErrNoError)
+ }
+ }
+ return metadataResponse
+}
+
+// MockOffsetResponse is an `OffsetResponse` builder.
+type MockOffsetResponse struct {
+ offsets map[string]map[int32]map[int64]int64
+ t TestReporter
+}
+
+func NewMockOffsetResponse(t TestReporter) *MockOffsetResponse {
+ return &MockOffsetResponse{
+ offsets: make(map[string]map[int32]map[int64]int64),
+ t: t,
+ }
+}
+
+func (mor *MockOffsetResponse) SetOffset(topic string, partition int32, time, offset int64) *MockOffsetResponse {
+ partitions := mor.offsets[topic]
+ if partitions == nil {
+ partitions = make(map[int32]map[int64]int64)
+ mor.offsets[topic] = partitions
+ }
+ times := partitions[partition]
+ if times == nil {
+ times = make(map[int64]int64)
+ partitions[partition] = times
+ }
+ times[time] = offset
+ return mor
+}
+
+func (mor *MockOffsetResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ offsetRequest := reqBody.(*OffsetRequest)
+ offsetResponse := &OffsetResponse{Version: offsetRequest.Version}
+ for topic, partitions := range offsetRequest.blocks {
+ for partition, block := range partitions {
+ offset := mor.getOffset(topic, partition, block.timestamp)
+ offsetResponse.AddTopicPartition(topic, partition, offset)
+ }
+ }
+ return offsetResponse
+}
+
+func (mor *MockOffsetResponse) getOffset(topic string, partition int32, time int64) int64 {
+ partitions := mor.offsets[topic]
+ if partitions == nil {
+ mor.t.Errorf("missing topic: %s", topic)
+ }
+ times := partitions[partition]
+ if times == nil {
+ mor.t.Errorf("missing partition: %d", partition)
+ }
+ offset, ok := times[time]
+ if !ok {
+ mor.t.Errorf("missing time: %d", time)
+ }
+ return offset
+}
+
+// mockMessage is a message that used to be mocked for `FetchResponse`
+type mockMessage struct {
+ key Encoder
+ msg Encoder
+}
+
+func newMockMessage(key, msg Encoder) *mockMessage {
+ return &mockMessage{
+ key: key,
+ msg: msg,
+ }
+}
+
+// MockFetchResponse is a `FetchResponse` builder.
+type MockFetchResponse struct {
+ messages map[string]map[int32]map[int64]*mockMessage
+ messagesLock *sync.RWMutex
+ highWaterMarks map[string]map[int32]int64
+ t TestReporter
+ batchSize int
+}
+
+func NewMockFetchResponse(t TestReporter, batchSize int) *MockFetchResponse {
+ return &MockFetchResponse{
+ messages: make(map[string]map[int32]map[int64]*mockMessage),
+ messagesLock: &sync.RWMutex{},
+ highWaterMarks: make(map[string]map[int32]int64),
+ t: t,
+ batchSize: batchSize,
+ }
+}
+
+func (mfr *MockFetchResponse) SetMessage(topic string, partition int32, offset int64, msg Encoder) *MockFetchResponse {
+ return mfr.SetMessageWithKey(topic, partition, offset, nil, msg)
+}
+
+func (mfr *MockFetchResponse) SetMessageWithKey(topic string, partition int32, offset int64, key, msg Encoder) *MockFetchResponse {
+ mfr.messagesLock.Lock()
+ defer mfr.messagesLock.Unlock()
+ partitions := mfr.messages[topic]
+ if partitions == nil {
+ partitions = make(map[int32]map[int64]*mockMessage)
+ mfr.messages[topic] = partitions
+ }
+ messages := partitions[partition]
+ if messages == nil {
+ messages = make(map[int64]*mockMessage)
+ partitions[partition] = messages
+ }
+ messages[offset] = newMockMessage(key, msg)
+ return mfr
+}
+
+func (mfr *MockFetchResponse) SetHighWaterMark(topic string, partition int32, offset int64) *MockFetchResponse {
+ partitions := mfr.highWaterMarks[topic]
+ if partitions == nil {
+ partitions = make(map[int32]int64)
+ mfr.highWaterMarks[topic] = partitions
+ }
+ partitions[partition] = offset
+ return mfr
+}
+
+func (mfr *MockFetchResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ fetchRequest := reqBody.(*FetchRequest)
+ res := &FetchResponse{
+ Version: fetchRequest.Version,
+ }
+ for topic, partitions := range fetchRequest.blocks {
+ for partition, block := range partitions {
+ initialOffset := block.fetchOffset
+ offset := initialOffset
+ maxOffset := initialOffset + int64(mfr.getMessageCount(topic, partition))
+ for i := 0; i < mfr.batchSize && offset < maxOffset; {
+ msg := mfr.getMessage(topic, partition, offset)
+ if msg != nil {
+ res.AddMessage(topic, partition, msg.key, msg.msg, offset)
+ i++
+ }
+ offset++
+ }
+ fb := res.GetBlock(topic, partition)
+ if fb == nil {
+ res.AddError(topic, partition, ErrNoError)
+ fb = res.GetBlock(topic, partition)
+ }
+ fb.HighWaterMarkOffset = mfr.getHighWaterMark(topic, partition)
+ }
+ }
+ return res
+}
+
+func (mfr *MockFetchResponse) getMessage(topic string, partition int32, offset int64) *mockMessage {
+ mfr.messagesLock.RLock()
+ defer mfr.messagesLock.RUnlock()
+ partitions := mfr.messages[topic]
+ if partitions == nil {
+ return nil
+ }
+ messages := partitions[partition]
+ if messages == nil {
+ return nil
+ }
+ return messages[offset]
+}
+
+func (mfr *MockFetchResponse) getMessageCount(topic string, partition int32) int {
+ mfr.messagesLock.RLock()
+ defer mfr.messagesLock.RUnlock()
+ partitions := mfr.messages[topic]
+ if partitions == nil {
+ return 0
+ }
+ messages := partitions[partition]
+ if messages == nil {
+ return 0
+ }
+ return len(messages)
+}
+
+func (mfr *MockFetchResponse) getHighWaterMark(topic string, partition int32) int64 {
+ partitions := mfr.highWaterMarks[topic]
+ if partitions == nil {
+ return 0
+ }
+ return partitions[partition]
+}
+
+// MockConsumerMetadataResponse is a `ConsumerMetadataResponse` builder.
+type MockConsumerMetadataResponse struct {
+ coordinators map[string]interface{}
+ t TestReporter
+}
+
+func NewMockConsumerMetadataResponse(t TestReporter) *MockConsumerMetadataResponse {
+ return &MockConsumerMetadataResponse{
+ coordinators: make(map[string]interface{}),
+ t: t,
+ }
+}
+
+func (mr *MockConsumerMetadataResponse) SetCoordinator(group string, broker *MockBroker) *MockConsumerMetadataResponse {
+ mr.coordinators[group] = broker
+ return mr
+}
+
+func (mr *MockConsumerMetadataResponse) SetError(group string, kerror KError) *MockConsumerMetadataResponse {
+ mr.coordinators[group] = kerror
+ return mr
+}
+
+func (mr *MockConsumerMetadataResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*ConsumerMetadataRequest)
+ group := req.ConsumerGroup
+ res := &ConsumerMetadataResponse{Version: req.version()}
+ v := mr.coordinators[group]
+ switch v := v.(type) {
+ case *MockBroker:
+ res.Coordinator = &Broker{id: v.BrokerID(), addr: v.Addr()}
+ case KError:
+ res.Err = v
+ }
+ return res
+}
+
+// MockFindCoordinatorResponse is a `FindCoordinatorResponse` builder.
+type MockFindCoordinatorResponse struct {
+ groupCoordinators map[string]interface{}
+ transCoordinators map[string]interface{}
+ t TestReporter
+}
+
+func NewMockFindCoordinatorResponse(t TestReporter) *MockFindCoordinatorResponse {
+ return &MockFindCoordinatorResponse{
+ groupCoordinators: make(map[string]interface{}),
+ transCoordinators: make(map[string]interface{}),
+ t: t,
+ }
+}
+
+func (mr *MockFindCoordinatorResponse) SetCoordinator(coordinatorType CoordinatorType, group string, broker *MockBroker) *MockFindCoordinatorResponse {
+ switch coordinatorType {
+ case CoordinatorGroup:
+ mr.groupCoordinators[group] = broker
+ case CoordinatorTransaction:
+ mr.transCoordinators[group] = broker
+ }
+ return mr
+}
+
+func (mr *MockFindCoordinatorResponse) SetError(coordinatorType CoordinatorType, group string, kerror KError) *MockFindCoordinatorResponse {
+ switch coordinatorType {
+ case CoordinatorGroup:
+ mr.groupCoordinators[group] = kerror
+ case CoordinatorTransaction:
+ mr.transCoordinators[group] = kerror
+ }
+ return mr
+}
+
+func (mr *MockFindCoordinatorResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*FindCoordinatorRequest)
+ res := &FindCoordinatorResponse{Version: req.version()}
+ var v interface{}
+ switch req.CoordinatorType {
+ case CoordinatorGroup:
+ v = mr.groupCoordinators[req.CoordinatorKey]
+ case CoordinatorTransaction:
+ v = mr.transCoordinators[req.CoordinatorKey]
+ }
+ switch v := v.(type) {
+ case *MockBroker:
+ res.Coordinator = &Broker{id: v.BrokerID(), addr: v.Addr()}
+ case KError:
+ res.Err = v
+ }
+ return res
+}
+
+// MockOffsetCommitResponse is a `OffsetCommitResponse` builder.
+type MockOffsetCommitResponse struct {
+ errors map[string]map[string]map[int32]KError
+ t TestReporter
+}
+
+func NewMockOffsetCommitResponse(t TestReporter) *MockOffsetCommitResponse {
+ return &MockOffsetCommitResponse{t: t}
+}
+
+func (mr *MockOffsetCommitResponse) SetError(group, topic string, partition int32, kerror KError) *MockOffsetCommitResponse {
+ if mr.errors == nil {
+ mr.errors = make(map[string]map[string]map[int32]KError)
+ }
+ topics := mr.errors[group]
+ if topics == nil {
+ topics = make(map[string]map[int32]KError)
+ mr.errors[group] = topics
+ }
+ partitions := topics[topic]
+ if partitions == nil {
+ partitions = make(map[int32]KError)
+ topics[topic] = partitions
+ }
+ partitions[partition] = kerror
+ return mr
+}
+
+func (mr *MockOffsetCommitResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*OffsetCommitRequest)
+ group := req.ConsumerGroup
+ res := &OffsetCommitResponse{Version: req.version()}
+ for topic, partitions := range req.blocks {
+ for partition := range partitions {
+ res.AddError(topic, partition, mr.getError(group, topic, partition))
+ }
+ }
+ return res
+}
+
+func (mr *MockOffsetCommitResponse) getError(group, topic string, partition int32) KError {
+ topics := mr.errors[group]
+ if topics == nil {
+ return ErrNoError
+ }
+ partitions := topics[topic]
+ if partitions == nil {
+ return ErrNoError
+ }
+ kerror, ok := partitions[partition]
+ if !ok {
+ return ErrNoError
+ }
+ return kerror
+}
+
+// MockProduceResponse is a `ProduceResponse` builder.
+type MockProduceResponse struct {
+ version int16
+ errors map[string]map[int32]KError
+ t TestReporter
+}
+
+func NewMockProduceResponse(t TestReporter) *MockProduceResponse {
+ return &MockProduceResponse{t: t}
+}
+
+func (mr *MockProduceResponse) SetVersion(version int16) *MockProduceResponse {
+ mr.version = version
+ return mr
+}
+
+func (mr *MockProduceResponse) SetError(topic string, partition int32, kerror KError) *MockProduceResponse {
+ if mr.errors == nil {
+ mr.errors = make(map[string]map[int32]KError)
+ }
+ partitions := mr.errors[topic]
+ if partitions == nil {
+ partitions = make(map[int32]KError)
+ mr.errors[topic] = partitions
+ }
+ partitions[partition] = kerror
+ return mr
+}
+
+func (mr *MockProduceResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*ProduceRequest)
+ res := &ProduceResponse{
+ Version: req.version(),
+ }
+ if mr.version > 0 {
+ res.Version = mr.version
+ }
+ for topic, partitions := range req.records {
+ for partition := range partitions {
+ res.AddTopicPartition(topic, partition, mr.getError(topic, partition))
+ }
+ }
+ return res
+}
+
+func (mr *MockProduceResponse) getError(topic string, partition int32) KError {
+ partitions := mr.errors[topic]
+ if partitions == nil {
+ return ErrNoError
+ }
+ kerror, ok := partitions[partition]
+ if !ok {
+ return ErrNoError
+ }
+ return kerror
+}
+
+// MockOffsetFetchResponse is a `OffsetFetchResponse` builder.
+type MockOffsetFetchResponse struct {
+ offsets map[string]map[string]map[int32]*OffsetFetchResponseBlock
+ error KError
+ t TestReporter
+}
+
+func NewMockOffsetFetchResponse(t TestReporter) *MockOffsetFetchResponse {
+ return &MockOffsetFetchResponse{t: t}
+}
+
+func (mr *MockOffsetFetchResponse) SetOffset(group, topic string, partition int32, offset int64, metadata string, kerror KError) *MockOffsetFetchResponse {
+ if mr.offsets == nil {
+ mr.offsets = make(map[string]map[string]map[int32]*OffsetFetchResponseBlock)
+ }
+ topics := mr.offsets[group]
+ if topics == nil {
+ topics = make(map[string]map[int32]*OffsetFetchResponseBlock)
+ mr.offsets[group] = topics
+ }
+ partitions := topics[topic]
+ if partitions == nil {
+ partitions = make(map[int32]*OffsetFetchResponseBlock)
+ topics[topic] = partitions
+ }
+ partitions[partition] = &OffsetFetchResponseBlock{offset, 0, metadata, kerror}
+ return mr
+}
+
+func (mr *MockOffsetFetchResponse) SetError(kerror KError) *MockOffsetFetchResponse {
+ mr.error = kerror
+ return mr
+}
+
+func (mr *MockOffsetFetchResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*OffsetFetchRequest)
+ group := req.ConsumerGroup
+ res := &OffsetFetchResponse{Version: req.Version}
+
+ for topic, partitions := range mr.offsets[group] {
+ for partition, block := range partitions {
+ res.AddBlock(topic, partition, block)
+ }
+ }
+
+ if res.Version >= 2 {
+ res.Err = mr.error
+ }
+ return res
+}
+
+type MockCreateTopicsResponse struct {
+ t TestReporter
+}
+
+func NewMockCreateTopicsResponse(t TestReporter) *MockCreateTopicsResponse {
+ return &MockCreateTopicsResponse{t: t}
+}
+
+func (mr *MockCreateTopicsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*CreateTopicsRequest)
+ res := &CreateTopicsResponse{
+ Version: req.Version,
+ }
+ res.TopicErrors = make(map[string]*TopicError)
+
+ for topic := range req.TopicDetails {
+ if res.Version >= 1 && strings.HasPrefix(topic, "_") {
+ msg := "insufficient permissions to create topic with reserved prefix"
+ res.TopicErrors[topic] = &TopicError{
+ Err: ErrTopicAuthorizationFailed,
+ ErrMsg: &msg,
+ }
+ continue
+ }
+ res.TopicErrors[topic] = &TopicError{Err: ErrNoError}
+ }
+ return res
+}
+
+type MockDeleteTopicsResponse struct {
+ t TestReporter
+ error KError
+}
+
+func NewMockDeleteTopicsResponse(t TestReporter) *MockDeleteTopicsResponse {
+ return &MockDeleteTopicsResponse{t: t}
+}
+
+func (mr *MockDeleteTopicsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*DeleteTopicsRequest)
+ res := &DeleteTopicsResponse{Version: req.version()}
+ res.TopicErrorCodes = make(map[string]KError)
+
+ for _, topic := range req.Topics {
+ res.TopicErrorCodes[topic] = mr.error
+ }
+ res.Version = req.Version
+ return res
+}
+
+func (mr *MockDeleteTopicsResponse) SetError(kerror KError) *MockDeleteTopicsResponse {
+ mr.error = kerror
+ return mr
+}
+
+type MockCreatePartitionsResponse struct {
+ t TestReporter
+}
+
+func NewMockCreatePartitionsResponse(t TestReporter) *MockCreatePartitionsResponse {
+ return &MockCreatePartitionsResponse{t: t}
+}
+
+func (mr *MockCreatePartitionsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*CreatePartitionsRequest)
+ res := &CreatePartitionsResponse{Version: req.version()}
+ res.TopicPartitionErrors = make(map[string]*TopicPartitionError)
+
+ for topic := range req.TopicPartitions {
+ if strings.HasPrefix(topic, "_") {
+ msg := "insufficient permissions to create partition on topic with reserved prefix"
+ res.TopicPartitionErrors[topic] = &TopicPartitionError{
+ Err: ErrTopicAuthorizationFailed,
+ ErrMsg: &msg,
+ }
+ continue
+ }
+ res.TopicPartitionErrors[topic] = &TopicPartitionError{Err: ErrNoError}
+ }
+ return res
+}
+
+type MockAlterPartitionReassignmentsResponse struct {
+ t TestReporter
+}
+
+func NewMockAlterPartitionReassignmentsResponse(t TestReporter) *MockAlterPartitionReassignmentsResponse {
+ return &MockAlterPartitionReassignmentsResponse{t: t}
+}
+
+func (mr *MockAlterPartitionReassignmentsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*AlterPartitionReassignmentsRequest)
+ _ = req
+ res := &AlterPartitionReassignmentsResponse{Version: req.version()}
+ return res
+}
+
+type MockListPartitionReassignmentsResponse struct {
+ t TestReporter
+}
+
+func NewMockListPartitionReassignmentsResponse(t TestReporter) *MockListPartitionReassignmentsResponse {
+ return &MockListPartitionReassignmentsResponse{t: t}
+}
+
+func (mr *MockListPartitionReassignmentsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*ListPartitionReassignmentsRequest)
+ _ = req
+ res := &ListPartitionReassignmentsResponse{Version: req.version()}
+
+ for topic, partitions := range req.blocks {
+ for _, partition := range partitions {
+ res.AddBlock(topic, partition, []int32{0}, []int32{1}, []int32{2})
+ }
+ }
+
+ return res
+}
+
+type MockElectLeadersResponse struct {
+ t TestReporter
+}
+
+func NewMockElectLeadersResponse(t TestReporter) *MockElectLeadersResponse {
+ return &MockElectLeadersResponse{t: t}
+}
+
+func (mr *MockElectLeadersResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*ElectLeadersRequest)
+ res := &ElectLeadersResponse{Version: req.version(), ReplicaElectionResults: map[string]map[int32]*PartitionResult{}}
+
+ for topic, partitions := range req.TopicPartitions {
+ for _, partition := range partitions {
+ res.ReplicaElectionResults[topic] = map[int32]*PartitionResult{
+ partition: {ErrorCode: ErrNoError},
+ }
+ }
+ }
+ return res
+}
+
+type MockDeleteRecordsResponse struct {
+ t TestReporter
+}
+
+func NewMockDeleteRecordsResponse(t TestReporter) *MockDeleteRecordsResponse {
+ return &MockDeleteRecordsResponse{t: t}
+}
+
+func (mr *MockDeleteRecordsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*DeleteRecordsRequest)
+ res := &DeleteRecordsResponse{Version: req.version()}
+ res.Topics = make(map[string]*DeleteRecordsResponseTopic)
+
+ for topic, deleteRecordRequestTopic := range req.Topics {
+ partitions := make(map[int32]*DeleteRecordsResponsePartition)
+ for partition := range deleteRecordRequestTopic.PartitionOffsets {
+ partitions[partition] = &DeleteRecordsResponsePartition{Err: ErrNoError}
+ }
+ res.Topics[topic] = &DeleteRecordsResponseTopic{Partitions: partitions}
+ }
+ return res
+}
+
+type MockDescribeConfigsResponse struct {
+ t TestReporter
+}
+
+func NewMockDescribeConfigsResponse(t TestReporter) *MockDescribeConfigsResponse {
+ return &MockDescribeConfigsResponse{t: t}
+}
+
+func (mr *MockDescribeConfigsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*DescribeConfigsRequest)
+ res := &DescribeConfigsResponse{
+ Version: req.Version,
+ }
+
+ includeSynonyms := req.Version > 0
+ includeSource := req.Version > 0
+
+ for _, r := range req.Resources {
+ var configEntries []*ConfigEntry
+ switch r.Type {
+ case BrokerResource:
+ configEntries = append(configEntries,
+ &ConfigEntry{
+ Name: "min.insync.replicas",
+ Value: "2",
+ ReadOnly: false,
+ Default: false,
+ },
+ )
+ res.Resources = append(res.Resources, &ResourceResponse{
+ Name: r.Name,
+ Configs: configEntries,
+ })
+ case BrokerLoggerResource:
+ configEntries = append(configEntries,
+ &ConfigEntry{
+ Name: "kafka.controller.KafkaController",
+ Value: "DEBUG",
+ ReadOnly: false,
+ Default: false,
+ },
+ )
+ res.Resources = append(res.Resources, &ResourceResponse{
+ Name: r.Name,
+ Configs: configEntries,
+ })
+ case TopicResource:
+ maxMessageBytes := &ConfigEntry{
+ Name: "max.message.bytes",
+ Value: "1000000",
+ ReadOnly: false,
+ Default: !includeSource,
+ Sensitive: false,
+ }
+ if includeSource {
+ maxMessageBytes.Source = SourceDefault
+ }
+ if includeSynonyms {
+ maxMessageBytes.Synonyms = []*ConfigSynonym{
+ {
+ ConfigName: "max.message.bytes",
+ ConfigValue: "500000",
+ },
+ }
+ }
+ retentionMs := &ConfigEntry{
+ Name: "retention.ms",
+ Value: "5000",
+ ReadOnly: false,
+ Default: false,
+ Sensitive: false,
+ }
+ if includeSynonyms {
+ retentionMs.Synonyms = []*ConfigSynonym{
+ {
+ ConfigName: "log.retention.ms",
+ ConfigValue: "2500",
+ },
+ }
+ }
+ password := &ConfigEntry{
+ Name: "password",
+ Value: "12345",
+ ReadOnly: false,
+ Default: false,
+ Sensitive: true,
+ }
+ configEntries = append(
+ configEntries, maxMessageBytes, retentionMs, password)
+ res.Resources = append(res.Resources, &ResourceResponse{
+ Name: r.Name,
+ Configs: configEntries,
+ })
+ }
+ }
+ return res
+}
+
+type MockDescribeConfigsResponseWithErrorCode struct {
+ t TestReporter
+}
+
+func NewMockDescribeConfigsResponseWithErrorCode(t TestReporter) *MockDescribeConfigsResponseWithErrorCode {
+ return &MockDescribeConfigsResponseWithErrorCode{t: t}
+}
+
+func (mr *MockDescribeConfigsResponseWithErrorCode) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*DescribeConfigsRequest)
+ res := &DescribeConfigsResponse{
+ Version: req.Version,
+ }
+
+ for _, r := range req.Resources {
+ res.Resources = append(res.Resources, &ResourceResponse{
+ Name: r.Name,
+ Type: r.Type,
+ ErrorCode: 83,
+ ErrorMsg: "",
+ })
+ }
+ return res
+}
+
+type MockAlterConfigsResponse struct {
+ t TestReporter
+}
+
+func NewMockAlterConfigsResponse(t TestReporter) *MockAlterConfigsResponse {
+ return &MockAlterConfigsResponse{t: t}
+}
+
+func (mr *MockAlterConfigsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*AlterConfigsRequest)
+ res := &AlterConfigsResponse{Version: req.version()}
+
+ for _, r := range req.Resources {
+ res.Resources = append(res.Resources, &AlterConfigsResourceResponse{
+ Name: r.Name,
+ Type: r.Type,
+ ErrorMsg: "",
+ })
+ }
+ return res
+}
+
+type MockAlterConfigsResponseWithErrorCode struct {
+ t TestReporter
+}
+
+func NewMockAlterConfigsResponseWithErrorCode(t TestReporter) *MockAlterConfigsResponseWithErrorCode {
+ return &MockAlterConfigsResponseWithErrorCode{t: t}
+}
+
+func (mr *MockAlterConfigsResponseWithErrorCode) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*AlterConfigsRequest)
+ res := &AlterConfigsResponse{Version: req.version()}
+
+ for _, r := range req.Resources {
+ res.Resources = append(res.Resources, &AlterConfigsResourceResponse{
+ Name: r.Name,
+ Type: r.Type,
+ ErrorCode: 83,
+ ErrorMsg: "",
+ })
+ }
+ return res
+}
+
+type MockIncrementalAlterConfigsResponse struct {
+ t TestReporter
+}
+
+func NewMockIncrementalAlterConfigsResponse(t TestReporter) *MockIncrementalAlterConfigsResponse {
+ return &MockIncrementalAlterConfigsResponse{t: t}
+}
+
+func (mr *MockIncrementalAlterConfigsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*IncrementalAlterConfigsRequest)
+ res := &IncrementalAlterConfigsResponse{Version: req.version()}
+
+ for _, r := range req.Resources {
+ res.Resources = append(res.Resources, &AlterConfigsResourceResponse{
+ Name: r.Name,
+ Type: r.Type,
+ ErrorMsg: "",
+ })
+ }
+ return res
+}
+
+type MockIncrementalAlterConfigsResponseWithErrorCode struct {
+ t TestReporter
+}
+
+func NewMockIncrementalAlterConfigsResponseWithErrorCode(t TestReporter) *MockIncrementalAlterConfigsResponseWithErrorCode {
+ return &MockIncrementalAlterConfigsResponseWithErrorCode{t: t}
+}
+
+func (mr *MockIncrementalAlterConfigsResponseWithErrorCode) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*IncrementalAlterConfigsRequest)
+ res := &IncrementalAlterConfigsResponse{Version: req.version()}
+
+ for _, r := range req.Resources {
+ res.Resources = append(res.Resources, &AlterConfigsResourceResponse{
+ Name: r.Name,
+ Type: r.Type,
+ ErrorCode: 83,
+ ErrorMsg: "",
+ })
+ }
+ return res
+}
+
+type MockCreateAclsResponse struct {
+ t TestReporter
+}
+
+func NewMockCreateAclsResponse(t TestReporter) *MockCreateAclsResponse {
+ return &MockCreateAclsResponse{t: t}
+}
+
+func (mr *MockCreateAclsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*CreateAclsRequest)
+ res := &CreateAclsResponse{Version: req.version()}
+
+ for range req.AclCreations {
+ res.AclCreationResponses = append(res.AclCreationResponses, &AclCreationResponse{Err: ErrNoError})
+ }
+ return res
+}
+
+type MockCreateAclsResponseError struct {
+ t TestReporter
+}
+
+func NewMockCreateAclsResponseWithError(t TestReporter) *MockCreateAclsResponseError {
+ return &MockCreateAclsResponseError{t: t}
+}
+
+func (mr *MockCreateAclsResponseError) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*CreateAclsRequest)
+ res := &CreateAclsResponse{Version: req.version()}
+
+ for range req.AclCreations {
+ res.AclCreationResponses = append(res.AclCreationResponses, &AclCreationResponse{Err: ErrInvalidRequest})
+ }
+ return res
+}
+
+type MockListAclsResponse struct {
+ t TestReporter
+}
+
+func NewMockListAclsResponse(t TestReporter) *MockListAclsResponse {
+ return &MockListAclsResponse{t: t}
+}
+
+func (mr *MockListAclsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*DescribeAclsRequest)
+ res := &DescribeAclsResponse{Version: req.version()}
+ res.Err = ErrNoError
+ acl := &ResourceAcls{}
+ if req.ResourceName != nil {
+ acl.Resource.ResourceName = *req.ResourceName
+ }
+ acl.Resource.ResourcePatternType = req.ResourcePatternTypeFilter
+ acl.Resource.ResourceType = req.ResourceType
+
+ host := "*"
+ if req.Host != nil {
+ host = *req.Host
+ }
+
+ principal := "User:test"
+ if req.Principal != nil {
+ principal = *req.Principal
+ }
+
+ permissionType := req.PermissionType
+ if permissionType == AclPermissionAny {
+ permissionType = AclPermissionAllow
+ }
+
+ acl.Acls = append(acl.Acls, &Acl{Operation: req.Operation, PermissionType: permissionType, Host: host, Principal: principal})
+ res.ResourceAcls = append(res.ResourceAcls, acl)
+ res.Version = int16(req.Version)
+ return res
+}
+
+type MockSaslAuthenticateResponse struct {
+ t TestReporter
+ kerror KError
+ saslAuthBytes []byte
+ sessionLifetimeMs int64
+}
+
+func NewMockSaslAuthenticateResponse(t TestReporter) *MockSaslAuthenticateResponse {
+ return &MockSaslAuthenticateResponse{t: t}
+}
+
+func (msar *MockSaslAuthenticateResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*SaslAuthenticateRequest)
+ res := &SaslAuthenticateResponse{
+ Version: req.version(),
+ Err: msar.kerror,
+ SaslAuthBytes: msar.saslAuthBytes,
+ SessionLifetimeMs: msar.sessionLifetimeMs,
+ }
+ return res
+}
+
+func (msar *MockSaslAuthenticateResponse) SetError(kerror KError) *MockSaslAuthenticateResponse {
+ msar.kerror = kerror
+ return msar
+}
+
+func (msar *MockSaslAuthenticateResponse) SetAuthBytes(saslAuthBytes []byte) *MockSaslAuthenticateResponse {
+ msar.saslAuthBytes = saslAuthBytes
+ return msar
+}
+
+func (msar *MockSaslAuthenticateResponse) SetSessionLifetimeMs(sessionLifetimeMs int64) *MockSaslAuthenticateResponse {
+ msar.sessionLifetimeMs = sessionLifetimeMs
+ return msar
+}
+
+type MockDeleteAclsResponse struct {
+ t TestReporter
+}
+
+type MockSaslHandshakeResponse struct {
+ enabledMechanisms []string
+ kerror KError
+ t TestReporter
+}
+
+func NewMockSaslHandshakeResponse(t TestReporter) *MockSaslHandshakeResponse {
+ return &MockSaslHandshakeResponse{t: t}
+}
+
+func (mshr *MockSaslHandshakeResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*SaslHandshakeRequest)
+ res := &SaslHandshakeResponse{Version: req.version()}
+ res.Err = mshr.kerror
+ res.EnabledMechanisms = mshr.enabledMechanisms
+ return res
+}
+
+func (mshr *MockSaslHandshakeResponse) SetError(kerror KError) *MockSaslHandshakeResponse {
+ mshr.kerror = kerror
+ return mshr
+}
+
+func (mshr *MockSaslHandshakeResponse) SetEnabledMechanisms(enabledMechanisms []string) *MockSaslHandshakeResponse {
+ mshr.enabledMechanisms = enabledMechanisms
+ return mshr
+}
+
+func NewMockDeleteAclsResponse(t TestReporter) *MockDeleteAclsResponse {
+ return &MockDeleteAclsResponse{t: t}
+}
+
+func (mr *MockDeleteAclsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*DeleteAclsRequest)
+ res := &DeleteAclsResponse{Version: req.version()}
+
+ for range req.Filters {
+ response := &FilterResponse{Err: ErrNoError}
+ response.MatchingAcls = append(response.MatchingAcls, &MatchingAcl{Err: ErrNoError})
+ res.FilterResponses = append(res.FilterResponses, response)
+ }
+ res.Version = int16(req.Version)
+ return res
+}
+
+type MockDeleteGroupsResponse struct {
+ deletedGroups []string
+}
+
+func NewMockDeleteGroupsRequest(t TestReporter) *MockDeleteGroupsResponse {
+ return &MockDeleteGroupsResponse{}
+}
+
+func (m *MockDeleteGroupsResponse) SetDeletedGroups(groups []string) *MockDeleteGroupsResponse {
+ m.deletedGroups = groups
+ return m
+}
+
+func (m *MockDeleteGroupsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*DeleteGroupsRequest)
+ resp := &DeleteGroupsResponse{
+ Version: req.version(),
+ GroupErrorCodes: map[string]KError{},
+ }
+ for _, group := range m.deletedGroups {
+ resp.GroupErrorCodes[group] = ErrNoError
+ }
+ return resp
+}
+
+type MockDeleteOffsetResponse struct {
+ errorCode KError
+ topic string
+ partition int32
+ errorPartition KError
+}
+
+func NewMockDeleteOffsetRequest(t TestReporter) *MockDeleteOffsetResponse {
+ return &MockDeleteOffsetResponse{}
+}
+
+func (m *MockDeleteOffsetResponse) SetDeletedOffset(errorCode KError, topic string, partition int32, errorPartition KError) *MockDeleteOffsetResponse {
+ m.errorCode = errorCode
+ m.topic = topic
+ m.partition = partition
+ m.errorPartition = errorPartition
+ return m
+}
+
+func (m *MockDeleteOffsetResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*DeleteOffsetsRequest)
+ resp := &DeleteOffsetsResponse{
+ Version: req.version(),
+ ErrorCode: m.errorCode,
+ Errors: map[string]map[int32]KError{
+ m.topic: {m.partition: m.errorPartition},
+ },
+ }
+ return resp
+}
+
+type MockJoinGroupResponse struct {
+ t TestReporter
+
+ ThrottleTime int32
+ Err KError
+ GenerationId int32
+ GroupProtocol string
+ LeaderId string
+ MemberId string
+ Members []GroupMember
+}
+
+func NewMockJoinGroupResponse(t TestReporter) *MockJoinGroupResponse {
+ return &MockJoinGroupResponse{
+ t: t,
+ Members: make([]GroupMember, 0),
+ }
+}
+
+func (m *MockJoinGroupResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*JoinGroupRequest)
+ resp := &JoinGroupResponse{
+ Version: req.Version,
+ ThrottleTime: m.ThrottleTime,
+ Err: m.Err,
+ GenerationId: m.GenerationId,
+ GroupProtocol: m.GroupProtocol,
+ LeaderId: m.LeaderId,
+ MemberId: m.MemberId,
+ Members: m.Members,
+ }
+ return resp
+}
+
+func (m *MockJoinGroupResponse) SetThrottleTime(t int32) *MockJoinGroupResponse {
+ m.ThrottleTime = t
+ return m
+}
+
+func (m *MockJoinGroupResponse) SetError(kerr KError) *MockJoinGroupResponse {
+ m.Err = kerr
+ return m
+}
+
+func (m *MockJoinGroupResponse) SetGenerationId(id int32) *MockJoinGroupResponse {
+ m.GenerationId = id
+ return m
+}
+
+func (m *MockJoinGroupResponse) SetGroupProtocol(proto string) *MockJoinGroupResponse {
+ m.GroupProtocol = proto
+ return m
+}
+
+func (m *MockJoinGroupResponse) SetLeaderId(id string) *MockJoinGroupResponse {
+ m.LeaderId = id
+ return m
+}
+
+func (m *MockJoinGroupResponse) SetMemberId(id string) *MockJoinGroupResponse {
+ m.MemberId = id
+ return m
+}
+
+func (m *MockJoinGroupResponse) SetMember(id string, meta *ConsumerGroupMemberMetadata) *MockJoinGroupResponse {
+ bin, err := encode(meta, nil)
+ if err != nil {
+ panic(fmt.Sprintf("error encoding member metadata: %v", err))
+ }
+ m.Members = append(m.Members, GroupMember{MemberId: id, Metadata: bin})
+ return m
+}
+
+type MockLeaveGroupResponse struct {
+ t TestReporter
+
+ Err KError
+}
+
+func NewMockLeaveGroupResponse(t TestReporter) *MockLeaveGroupResponse {
+ return &MockLeaveGroupResponse{t: t}
+}
+
+func (m *MockLeaveGroupResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*LeaveGroupRequest)
+ resp := &LeaveGroupResponse{
+ Version: req.version(),
+ Err: m.Err,
+ }
+ return resp
+}
+
+func (m *MockLeaveGroupResponse) SetError(kerr KError) *MockLeaveGroupResponse {
+ m.Err = kerr
+ return m
+}
+
+type MockSyncGroupResponse struct {
+ t TestReporter
+
+ Err KError
+ MemberAssignment []byte
+}
+
+func NewMockSyncGroupResponse(t TestReporter) *MockSyncGroupResponse {
+ return &MockSyncGroupResponse{t: t}
+}
+
+func (m *MockSyncGroupResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*SyncGroupRequest)
+ resp := &SyncGroupResponse{
+ Version: req.version(),
+ Err: m.Err,
+ MemberAssignment: m.MemberAssignment,
+ }
+ return resp
+}
+
+func (m *MockSyncGroupResponse) SetError(kerr KError) *MockSyncGroupResponse {
+ m.Err = kerr
+ return m
+}
+
+func (m *MockSyncGroupResponse) SetMemberAssignment(assignment *ConsumerGroupMemberAssignment) *MockSyncGroupResponse {
+ bin, err := encode(assignment, nil)
+ if err != nil {
+ panic(fmt.Sprintf("error encoding member assignment: %v", err))
+ }
+ m.MemberAssignment = bin
+ return m
+}
+
+type MockHeartbeatResponse struct {
+ t TestReporter
+
+ Err KError
+}
+
+func NewMockHeartbeatResponse(t TestReporter) *MockHeartbeatResponse {
+ return &MockHeartbeatResponse{t: t}
+}
+
+func (m *MockHeartbeatResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*HeartbeatRequest)
+ resp := &HeartbeatResponse{
+ Version: req.version(),
+ }
+ return resp
+}
+
+func (m *MockHeartbeatResponse) SetError(kerr KError) *MockHeartbeatResponse {
+ m.Err = kerr
+ return m
+}
+
+type MockDescribeLogDirsResponse struct {
+ t TestReporter
+ logDirs []DescribeLogDirsResponseDirMetadata
+}
+
+func NewMockDescribeLogDirsResponse(t TestReporter) *MockDescribeLogDirsResponse {
+ return &MockDescribeLogDirsResponse{t: t}
+}
+
+func (m *MockDescribeLogDirsResponse) SetLogDirs(logDirPath string, topicPartitions map[string]int) *MockDescribeLogDirsResponse {
+ var topics []DescribeLogDirsResponseTopic
+ for topic := range topicPartitions {
+ var partitions []DescribeLogDirsResponsePartition
+ for i := 0; i < topicPartitions[topic]; i++ {
+ partitions = append(partitions, DescribeLogDirsResponsePartition{
+ PartitionID: int32(i),
+ IsTemporary: false,
+ OffsetLag: int64(0),
+ Size: int64(1234),
+ })
+ }
+ topics = append(topics, DescribeLogDirsResponseTopic{
+ Topic: topic,
+ Partitions: partitions,
+ })
+ }
+ logDir := DescribeLogDirsResponseDirMetadata{
+ ErrorCode: ErrNoError,
+ Path: logDirPath,
+ Topics: topics,
+ }
+ m.logDirs = []DescribeLogDirsResponseDirMetadata{logDir}
+ return m
+}
+
+func (m *MockDescribeLogDirsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*DescribeLogDirsRequest)
+ resp := &DescribeLogDirsResponse{
+ Version: req.version(),
+ LogDirs: m.logDirs,
+ }
+ return resp
+}
+
+type MockApiVersionsResponse struct {
+ t TestReporter
+ apiKeys []ApiVersionsResponseKey
+}
+
+func NewMockApiVersionsResponse(t TestReporter) *MockApiVersionsResponse {
+ return &MockApiVersionsResponse{
+ t: t,
+ apiKeys: []ApiVersionsResponseKey{
+ {
+ ApiKey: 0,
+ MinVersion: 5,
+ MaxVersion: 8,
+ },
+ {
+ ApiKey: 1,
+ MinVersion: 7,
+ MaxVersion: 11,
+ },
+ },
+ }
+}
+
+func (m *MockApiVersionsResponse) SetApiKeys(apiKeys []ApiVersionsResponseKey) *MockApiVersionsResponse {
+ m.apiKeys = apiKeys
+ return m
+}
+
+func (m *MockApiVersionsResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*ApiVersionsRequest)
+ res := &ApiVersionsResponse{
+ Version: req.Version,
+ ApiKeys: m.apiKeys,
+ }
+ return res
+}
+
+// MockInitProducerIDResponse is an `InitPorducerIDResponse` builder.
+type MockInitProducerIDResponse struct {
+ producerID int64
+ producerEpoch int16
+ err KError
+ t TestReporter
+}
+
+func NewMockInitProducerIDResponse(t TestReporter) *MockInitProducerIDResponse {
+ return &MockInitProducerIDResponse{
+ t: t,
+ }
+}
+
+func (m *MockInitProducerIDResponse) SetProducerID(id int) *MockInitProducerIDResponse {
+ m.producerID = int64(id)
+ return m
+}
+
+func (m *MockInitProducerIDResponse) SetProducerEpoch(epoch int) *MockInitProducerIDResponse {
+ m.producerEpoch = int16(epoch)
+ return m
+}
+
+func (m *MockInitProducerIDResponse) SetError(err KError) *MockInitProducerIDResponse {
+ m.err = err
+ return m
+}
+
+func (m *MockInitProducerIDResponse) For(reqBody versionedDecoder) encoderWithHeader {
+ req := reqBody.(*InitProducerIDRequest)
+ res := &InitProducerIDResponse{
+ Version: req.Version,
+ Err: m.err,
+ ProducerID: m.producerID,
+ ProducerEpoch: m.producerEpoch,
+ }
+ return res
+}
diff --git a/vendor/github.com/IBM/sarama/offset_commit_request.go b/vendor/github.com/IBM/sarama/offset_commit_request.go
new file mode 100644
index 0000000..2c5c693
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/offset_commit_request.go
@@ -0,0 +1,257 @@
+package sarama
+
+import "errors"
+
+// ReceiveTime is a special value for the timestamp field of Offset Commit Requests which
+// tells the broker to set the timestamp to the time at which the request was received.
+// The timestamp is only used if message version 1 is used, which requires kafka 0.8.2.
+const ReceiveTime int64 = -1
+
+// GroupGenerationUndefined is a special value for the group generation field of
+// Offset Commit Requests that should be used when a consumer group does not rely
+// on Kafka for partition management.
+const GroupGenerationUndefined = -1
+
+type offsetCommitRequestBlock struct {
+ offset int64
+ timestamp int64
+ committedLeaderEpoch int32
+ metadata string
+}
+
+func (b *offsetCommitRequestBlock) encode(pe packetEncoder, version int16) error {
+ pe.putInt64(b.offset)
+ if version == 1 {
+ pe.putInt64(b.timestamp)
+ } else if b.timestamp != 0 {
+ Logger.Println("Non-zero timestamp specified for OffsetCommitRequest not v1, it will be ignored")
+ }
+ if version >= 6 {
+ pe.putInt32(b.committedLeaderEpoch)
+ }
+
+ return pe.putString(b.metadata)
+}
+
+func (b *offsetCommitRequestBlock) decode(pd packetDecoder, version int16) (err error) {
+ if b.offset, err = pd.getInt64(); err != nil {
+ return err
+ }
+ if version == 1 {
+ if b.timestamp, err = pd.getInt64(); err != nil {
+ return err
+ }
+ }
+ if version >= 6 {
+ if b.committedLeaderEpoch, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+
+ b.metadata, err = pd.getString()
+ return err
+}
+
+type OffsetCommitRequest struct {
+ ConsumerGroup string
+ ConsumerGroupGeneration int32 // v1 or later
+ ConsumerID string // v1 or later
+ GroupInstanceId *string // v7 or later
+ RetentionTime int64 // v2 or later
+
+ // Version can be:
+ // - 0 (kafka 0.8.1 and later)
+ // - 1 (kafka 0.8.2 and later)
+ // - 2 (kafka 0.9.0 and later)
+ // - 3 (kafka 0.11.0 and later)
+ // - 4 (kafka 2.0.0 and later)
+ // - 5&6 (kafka 2.1.0 and later)
+ // - 7 (kafka 2.3.0 and later)
+ Version int16
+ blocks map[string]map[int32]*offsetCommitRequestBlock
+}
+
+func (r *OffsetCommitRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *OffsetCommitRequest) encode(pe packetEncoder) error {
+ if r.Version < 0 || r.Version > 7 {
+ return PacketEncodingError{"invalid or unsupported OffsetCommitRequest version field"}
+ }
+
+ if err := pe.putString(r.ConsumerGroup); err != nil {
+ return err
+ }
+
+ if r.Version >= 1 {
+ pe.putInt32(r.ConsumerGroupGeneration)
+ if err := pe.putString(r.ConsumerID); err != nil {
+ return err
+ }
+ } else {
+ if r.ConsumerGroupGeneration != 0 {
+ Logger.Println("Non-zero ConsumerGroupGeneration specified for OffsetCommitRequest v0, it will be ignored")
+ }
+ if r.ConsumerID != "" {
+ Logger.Println("Non-empty ConsumerID specified for OffsetCommitRequest v0, it will be ignored")
+ }
+ }
+
+ // Version 5 removes RetentionTime, which is now controlled only by a broker configuration.
+ if r.Version >= 2 && r.Version <= 4 {
+ pe.putInt64(r.RetentionTime)
+ } else if r.RetentionTime != 0 {
+ Logger.Println("Non-zero RetentionTime specified for OffsetCommitRequest version <2, it will be ignored")
+ }
+
+ if r.Version >= 7 {
+ if err := pe.putNullableString(r.GroupInstanceId); err != nil {
+ return err
+ }
+ }
+
+ if err := pe.putArrayLength(len(r.blocks)); err != nil {
+ return err
+ }
+ for topic, partitions := range r.blocks {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putArrayLength(len(partitions)); err != nil {
+ return err
+ }
+ for partition, block := range partitions {
+ pe.putInt32(partition)
+ if err := block.encode(pe, r.Version); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func (r *OffsetCommitRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+
+ if r.ConsumerGroup, err = pd.getString(); err != nil {
+ return err
+ }
+
+ if r.Version >= 1 {
+ if r.ConsumerGroupGeneration, err = pd.getInt32(); err != nil {
+ return err
+ }
+ if r.ConsumerID, err = pd.getString(); err != nil {
+ return err
+ }
+ }
+
+ // Version 5 removes RetentionTime, which is now controlled only by a broker configuration.
+ if r.Version >= 2 && r.Version <= 4 {
+ if r.RetentionTime, err = pd.getInt64(); err != nil {
+ return err
+ }
+ }
+
+ if r.Version >= 7 {
+ if r.GroupInstanceId, err = pd.getNullableString(); err != nil {
+ return err
+ }
+ }
+
+ topicCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if topicCount == 0 {
+ return nil
+ }
+ r.blocks = make(map[string]map[int32]*offsetCommitRequestBlock)
+ for i := 0; i < topicCount; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ partitionCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ r.blocks[topic] = make(map[int32]*offsetCommitRequestBlock)
+ for j := 0; j < partitionCount; j++ {
+ partition, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ block := &offsetCommitRequestBlock{}
+ if err := block.decode(pd, r.Version); err != nil {
+ return err
+ }
+ r.blocks[topic][partition] = block
+ }
+ }
+ return nil
+}
+
+func (r *OffsetCommitRequest) key() int16 {
+ return apiKeyOffsetCommit
+}
+
+func (r *OffsetCommitRequest) version() int16 {
+ return r.Version
+}
+
+func (r *OffsetCommitRequest) headerVersion() int16 {
+ return 1
+}
+
+func (r *OffsetCommitRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 7
+}
+
+func (r *OffsetCommitRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 7:
+ return V2_3_0_0
+ case 5, 6:
+ return V2_1_0_0
+ case 4:
+ return V2_0_0_0
+ case 3:
+ return V0_11_0_0
+ case 2:
+ return V0_9_0_0
+ case 0, 1:
+ return V0_8_2_0
+ default:
+ return V2_4_0_0
+ }
+}
+
+func (r *OffsetCommitRequest) AddBlock(topic string, partitionID int32, offset int64, timestamp int64, metadata string) {
+ r.AddBlockWithLeaderEpoch(topic, partitionID, offset, 0, timestamp, metadata)
+}
+
+func (r *OffsetCommitRequest) AddBlockWithLeaderEpoch(topic string, partitionID int32, offset int64, leaderEpoch int32, timestamp int64, metadata string) {
+ if r.blocks == nil {
+ r.blocks = make(map[string]map[int32]*offsetCommitRequestBlock)
+ }
+
+ if r.blocks[topic] == nil {
+ r.blocks[topic] = make(map[int32]*offsetCommitRequestBlock)
+ }
+
+ r.blocks[topic][partitionID] = &offsetCommitRequestBlock{offset, timestamp, leaderEpoch, metadata}
+}
+
+func (r *OffsetCommitRequest) Offset(topic string, partitionID int32) (int64, string, error) {
+ partitions := r.blocks[topic]
+ if partitions == nil {
+ return 0, "", errors.New("no such offset")
+ }
+ block := partitions[partitionID]
+ if block == nil {
+ return 0, "", errors.New("no such offset")
+ }
+ return block.offset, block.metadata, nil
+}
diff --git a/vendor/github.com/IBM/sarama/offset_commit_response.go b/vendor/github.com/IBM/sarama/offset_commit_response.go
new file mode 100644
index 0000000..2af478c
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/offset_commit_response.go
@@ -0,0 +1,131 @@
+package sarama
+
+import "time"
+
+type OffsetCommitResponse struct {
+ Version int16
+ ThrottleTimeMs int32
+ Errors map[string]map[int32]KError
+}
+
+func (r *OffsetCommitResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *OffsetCommitResponse) AddError(topic string, partition int32, kerror KError) {
+ if r.Errors == nil {
+ r.Errors = make(map[string]map[int32]KError)
+ }
+ partitions := r.Errors[topic]
+ if partitions == nil {
+ partitions = make(map[int32]KError)
+ r.Errors[topic] = partitions
+ }
+ partitions[partition] = kerror
+}
+
+func (r *OffsetCommitResponse) encode(pe packetEncoder) error {
+ if r.Version >= 3 {
+ pe.putInt32(r.ThrottleTimeMs)
+ }
+ if err := pe.putArrayLength(len(r.Errors)); err != nil {
+ return err
+ }
+ for topic, partitions := range r.Errors {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putArrayLength(len(partitions)); err != nil {
+ return err
+ }
+ for partition, kerror := range partitions {
+ pe.putInt32(partition)
+ pe.putKError(kerror)
+ }
+ }
+ return nil
+}
+
+func (r *OffsetCommitResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+
+ if version >= 3 {
+ r.ThrottleTimeMs, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+ }
+
+ numTopics, err := pd.getArrayLength()
+ if err != nil || numTopics == 0 {
+ return err
+ }
+
+ r.Errors = make(map[string]map[int32]KError, numTopics)
+ for i := 0; i < numTopics; i++ {
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ numErrors, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Errors[name] = make(map[int32]KError, numErrors)
+
+ for j := 0; j < numErrors; j++ {
+ id, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ r.Errors[name][id], err = pd.getKError()
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func (r *OffsetCommitResponse) key() int16 {
+ return apiKeyOffsetCommit
+}
+
+func (r *OffsetCommitResponse) version() int16 {
+ return r.Version
+}
+
+func (r *OffsetCommitResponse) headerVersion() int16 {
+ return 0
+}
+
+func (r *OffsetCommitResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 7
+}
+
+func (r *OffsetCommitResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 7:
+ return V2_3_0_0
+ case 5, 6:
+ return V2_1_0_0
+ case 4:
+ return V2_0_0_0
+ case 3:
+ return V0_11_0_0
+ case 2:
+ return V0_9_0_0
+ case 0, 1:
+ return V0_8_2_0
+ default:
+ return V2_4_0_0
+ }
+}
+
+func (r *OffsetCommitResponse) throttleTime() time.Duration {
+ return time.Duration(r.ThrottleTimeMs) * time.Millisecond
+}
diff --git a/vendor/github.com/IBM/sarama/offset_fetch_request.go b/vendor/github.com/IBM/sarama/offset_fetch_request.go
new file mode 100644
index 0000000..0cfbe7f
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/offset_fetch_request.go
@@ -0,0 +1,206 @@
+package sarama
+
+type OffsetFetchRequest struct {
+ Version int16
+ ConsumerGroup string
+ RequireStable bool // requires v7+
+ partitions map[string][]int32
+}
+
+func (r *OffsetFetchRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func NewOffsetFetchRequest(
+ version KafkaVersion,
+ group string,
+ partitions map[string][]int32,
+) *OffsetFetchRequest {
+ request := &OffsetFetchRequest{
+ ConsumerGroup: group,
+ partitions: partitions,
+ }
+ if version.IsAtLeast(V2_5_0_0) {
+ // Version 7 is adding the require stable flag.
+ request.Version = 7
+ } else if version.IsAtLeast(V2_4_0_0) {
+ // Version 6 is the first flexible version.
+ request.Version = 6
+ } else if version.IsAtLeast(V2_1_0_0) {
+ // Version 3, 4, and 5 are the same as version 2.
+ request.Version = 5
+ } else if version.IsAtLeast(V2_0_0_0) {
+ request.Version = 4
+ } else if version.IsAtLeast(V0_11_0_0) {
+ request.Version = 3
+ } else if version.IsAtLeast(V0_10_2_0) {
+ // Starting in version 2, the request can contain a null topics array to indicate that offsets
+ // for all topics should be fetched. It also returns a top level error code
+ // for group or coordinator level errors.
+ request.Version = 2
+ } else if version.IsAtLeast(V0_8_2_0) {
+ // In version 0, the request read offsets from ZK.
+ //
+ // Starting in version 1, the broker supports fetching offsets from the internal __consumer_offsets topic.
+ request.Version = 1
+ }
+
+ return request
+}
+
+func (r *OffsetFetchRequest) encode(pe packetEncoder) (err error) {
+ if r.Version < 0 || r.Version > 7 {
+ return PacketEncodingError{"invalid or unsupported OffsetFetchRequest version field"}
+ }
+
+ err = pe.putString(r.ConsumerGroup)
+ if err != nil {
+ return err
+ }
+
+ if r.partitions == nil && r.Version >= 2 {
+ if err := pe.putArrayLength(-1); err != nil {
+ return err
+ }
+ } else {
+ if err = pe.putArrayLength(len(r.partitions)); err != nil {
+ return err
+ }
+ }
+
+ for topic, partitions := range r.partitions {
+ err = pe.putString(topic)
+ if err != nil {
+ return err
+ }
+
+ err = pe.putInt32Array(partitions)
+ if err != nil {
+ return err
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ }
+
+ if r.RequireStable && r.Version < 7 {
+ return PacketEncodingError{"requireStable is not supported. use version 7 or later"}
+ }
+
+ if r.Version >= 7 {
+ pe.putBool(r.RequireStable)
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *OffsetFetchRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ r.ConsumerGroup, err = pd.getString()
+ if err != nil {
+ return err
+ }
+
+ partitionCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if (partitionCount == 0 && version < 2) || partitionCount < 0 {
+ return nil
+ }
+
+ r.partitions = make(map[string][]int32, partitionCount)
+ for i := 0; i < partitionCount; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ partitions, err := pd.getInt32Array()
+ if err != nil {
+ return err
+ }
+ _, err = pd.getEmptyTaggedFieldArray()
+ if err != nil {
+ return err
+ }
+
+ r.partitions[topic] = partitions
+ }
+
+ if r.Version >= 7 {
+ r.RequireStable, err = pd.getBool()
+ if err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *OffsetFetchRequest) key() int16 {
+ return apiKeyOffsetFetch
+}
+
+func (r *OffsetFetchRequest) version() int16 {
+ return r.Version
+}
+
+func (r *OffsetFetchRequest) headerVersion() int16 {
+ if r.Version >= 6 {
+ return 2
+ }
+
+ return 1
+}
+
+func (r *OffsetFetchRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 7
+}
+
+func (r *OffsetFetchRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *OffsetFetchRequest) isFlexibleVersion(version int16) bool {
+ return version >= 6
+}
+
+func (r *OffsetFetchRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 7:
+ return V2_5_0_0
+ case 6:
+ return V2_4_0_0
+ case 5:
+ return V2_1_0_0
+ case 4:
+ return V2_0_0_0
+ case 3:
+ return V0_11_0_0
+ case 2:
+ return V0_10_2_0
+ case 1:
+ return V0_8_2_0
+ case 0:
+ return V0_8_2_0
+ default:
+ return V2_5_0_0
+ }
+}
+
+func (r *OffsetFetchRequest) ZeroPartitions() {
+ if r.partitions == nil && r.Version >= 2 {
+ r.partitions = make(map[string][]int32)
+ }
+}
+
+func (r *OffsetFetchRequest) AddPartition(topic string, partitionID int32) {
+ if r.partitions == nil {
+ r.partitions = make(map[string][]int32)
+ }
+
+ r.partitions[topic] = append(r.partitions[topic], partitionID)
+}
diff --git a/vendor/github.com/IBM/sarama/offset_fetch_response.go b/vendor/github.com/IBM/sarama/offset_fetch_response.go
new file mode 100644
index 0000000..0c27563
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/offset_fetch_response.go
@@ -0,0 +1,244 @@
+package sarama
+
+import "time"
+
+type OffsetFetchResponseBlock struct {
+ Offset int64
+ LeaderEpoch int32
+ Metadata string
+ Err KError
+}
+
+func (b *OffsetFetchResponseBlock) decode(pd packetDecoder, version int16) (err error) {
+ b.Offset, err = pd.getInt64()
+ if err != nil {
+ return err
+ }
+
+ if version >= 5 {
+ b.LeaderEpoch, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+ } else {
+ b.LeaderEpoch = -1
+ }
+
+ b.Metadata, err = pd.getString()
+ if err != nil {
+ return err
+ }
+
+ b.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (b *OffsetFetchResponseBlock) encode(pe packetEncoder, version int16) (err error) {
+ pe.putInt64(b.Offset)
+
+ if version >= 5 {
+ pe.putInt32(b.LeaderEpoch)
+ }
+ err = pe.putString(b.Metadata)
+ if err != nil {
+ return err
+ }
+
+ pe.putKError(b.Err)
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+type OffsetFetchResponse struct {
+ Version int16
+ ThrottleTimeMs int32
+ Blocks map[string]map[int32]*OffsetFetchResponseBlock
+ Err KError
+}
+
+func (r *OffsetFetchResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *OffsetFetchResponse) encode(pe packetEncoder) (err error) {
+ if r.Version >= 3 {
+ pe.putInt32(r.ThrottleTimeMs)
+ }
+ err = pe.putArrayLength(len(r.Blocks))
+ if err != nil {
+ return err
+ }
+
+ for topic, partitions := range r.Blocks {
+ err = pe.putString(topic)
+ if err != nil {
+ return err
+ }
+
+ err = pe.putArrayLength(len(partitions))
+ if err != nil {
+ return err
+ }
+ for partition, block := range partitions {
+ pe.putInt32(partition)
+ if err := block.encode(pe, r.Version); err != nil {
+ return err
+ }
+ }
+ pe.putEmptyTaggedFieldArray()
+ }
+ if r.Version >= 2 {
+ pe.putKError(r.Err)
+ }
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *OffsetFetchResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+
+ if version >= 3 {
+ r.ThrottleTimeMs, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+ }
+
+ numTopics, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ if numTopics > 0 {
+ r.Blocks = make(map[string]map[int32]*OffsetFetchResponseBlock, numTopics)
+ for i := 0; i < numTopics; i++ {
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ numBlocks, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Blocks[name] = nil
+ if numBlocks > 0 {
+ r.Blocks[name] = make(map[int32]*OffsetFetchResponseBlock, numBlocks)
+ }
+ for j := 0; j < numBlocks; j++ {
+ id, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ block := new(OffsetFetchResponseBlock)
+ err = block.decode(pd, version)
+ if err != nil {
+ return err
+ }
+
+ r.Blocks[name][id] = block
+ }
+
+ if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
+ return err
+ }
+ }
+ }
+
+ if version >= 2 {
+ r.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *OffsetFetchResponse) key() int16 {
+ return apiKeyOffsetFetch
+}
+
+func (r *OffsetFetchResponse) version() int16 {
+ return r.Version
+}
+
+func (r *OffsetFetchResponse) headerVersion() int16 {
+ if r.Version >= 6 {
+ return 1
+ }
+
+ return 0
+}
+
+func (r *OffsetFetchResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 7
+}
+
+func (r *OffsetFetchResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *OffsetFetchResponse) isFlexibleVersion(version int16) bool {
+ return version >= 6
+}
+
+func (r *OffsetFetchResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 7:
+ return V2_5_0_0
+ case 6:
+ return V2_4_0_0
+ case 5:
+ return V2_1_0_0
+ case 4:
+ return V2_0_0_0
+ case 3:
+ return V0_11_0_0
+ case 2:
+ return V0_10_2_0
+ case 1:
+ return V0_8_2_0
+ case 0:
+ return V0_8_2_0
+ default:
+ return V2_5_0_0
+ }
+}
+
+func (r *OffsetFetchResponse) throttleTime() time.Duration {
+ return time.Duration(r.ThrottleTimeMs) * time.Millisecond
+}
+
+func (r *OffsetFetchResponse) GetBlock(topic string, partition int32) *OffsetFetchResponseBlock {
+ if r.Blocks == nil {
+ return nil
+ }
+
+ if r.Blocks[topic] == nil {
+ return nil
+ }
+
+ return r.Blocks[topic][partition]
+}
+
+func (r *OffsetFetchResponse) AddBlock(topic string, partition int32, block *OffsetFetchResponseBlock) {
+ if r.Blocks == nil {
+ r.Blocks = make(map[string]map[int32]*OffsetFetchResponseBlock)
+ }
+ partitions := r.Blocks[topic]
+ if partitions == nil {
+ partitions = make(map[int32]*OffsetFetchResponseBlock)
+ r.Blocks[topic] = partitions
+ }
+ partitions[partition] = block
+}
diff --git a/vendor/github.com/IBM/sarama/offset_manager.go b/vendor/github.com/IBM/sarama/offset_manager.go
new file mode 100644
index 0000000..0b594c2
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/offset_manager.go
@@ -0,0 +1,663 @@
+package sarama
+
+import (
+ "sync"
+ "time"
+)
+
+// Offset Manager
+
+// OffsetManager uses Kafka to store and fetch consumed partition offsets.
+type OffsetManager interface {
+ // ManagePartition creates a PartitionOffsetManager on the given topic/partition.
+ // It will return an error if this OffsetManager is already managing the given
+ // topic/partition.
+ ManagePartition(topic string, partition int32) (PartitionOffsetManager, error)
+
+ // Close stops the OffsetManager from managing offsets. It is required to call
+ // this function before an OffsetManager object passes out of scope, as it
+ // will otherwise leak memory. You must call this after all the
+ // PartitionOffsetManagers are closed.
+ Close() error
+
+ // Commit commits the offsets. This method can be used if AutoCommit.Enable is
+ // set to false.
+ Commit()
+}
+
+type offsetManager struct {
+ client Client
+ conf *Config
+ group string
+ ticker *time.Ticker
+ sessionCanceler func()
+
+ memberID string
+ groupInstanceId *string
+ generation int32
+
+ broker *Broker
+ brokerLock sync.RWMutex
+
+ poms map[string]map[int32]*partitionOffsetManager
+ pomsLock sync.RWMutex
+
+ closeOnce sync.Once
+ closing chan none
+ closed chan none
+}
+
+// NewOffsetManagerFromClient creates a new OffsetManager from the given client.
+// It is still necessary to call Close() on the underlying client when finished with the partition manager.
+func NewOffsetManagerFromClient(group string, client Client) (OffsetManager, error) {
+ return newOffsetManagerFromClient(group, "", GroupGenerationUndefined, client, nil)
+}
+
+func newOffsetManagerFromClient(group, memberID string, generation int32, client Client, sessionCanceler func()) (*offsetManager, error) {
+ // Check that we are not dealing with a closed Client before processing any other arguments
+ if client.Closed() {
+ return nil, ErrClosedClient
+ }
+
+ conf := client.Config()
+ om := &offsetManager{
+ client: client,
+ conf: conf,
+ group: group,
+ poms: make(map[string]map[int32]*partitionOffsetManager),
+ sessionCanceler: sessionCanceler,
+
+ memberID: memberID,
+ generation: generation,
+
+ closing: make(chan none),
+ closed: make(chan none),
+ }
+ if conf.Consumer.Group.InstanceId != "" {
+ om.groupInstanceId = &conf.Consumer.Group.InstanceId
+ }
+ if conf.Consumer.Offsets.AutoCommit.Enable {
+ om.ticker = time.NewTicker(conf.Consumer.Offsets.AutoCommit.Interval)
+ go withRecover(om.mainLoop)
+ }
+
+ return om, nil
+}
+
+func (om *offsetManager) ManagePartition(topic string, partition int32) (PartitionOffsetManager, error) {
+ pom, err := om.newPartitionOffsetManager(topic, partition)
+ if err != nil {
+ return nil, err
+ }
+
+ om.pomsLock.Lock()
+ defer om.pomsLock.Unlock()
+
+ topicManagers := om.poms[topic]
+ if topicManagers == nil {
+ topicManagers = make(map[int32]*partitionOffsetManager)
+ om.poms[topic] = topicManagers
+ }
+
+ if topicManagers[partition] != nil {
+ return nil, ConfigurationError("That topic/partition is already being managed")
+ }
+
+ topicManagers[partition] = pom
+ return pom, nil
+}
+
+func (om *offsetManager) Close() error {
+ om.closeOnce.Do(func() {
+ // exit the mainLoop
+ close(om.closing)
+ if om.conf.Consumer.Offsets.AutoCommit.Enable {
+ <-om.closed
+ }
+
+ // mark all POMs as closed
+ om.asyncClosePOMs()
+
+ // flush one last time
+ if om.conf.Consumer.Offsets.AutoCommit.Enable {
+ for attempt := 0; attempt <= om.conf.Consumer.Offsets.Retry.Max; attempt++ {
+ om.flushToBroker()
+ if om.releasePOMs(false) == 0 {
+ break
+ }
+ }
+ }
+
+ om.releasePOMs(true)
+ om.brokerLock.Lock()
+ om.broker = nil
+ om.brokerLock.Unlock()
+ })
+ return nil
+}
+
+func (om *offsetManager) computeBackoff(retries int) time.Duration {
+ if om.conf.Metadata.Retry.BackoffFunc != nil {
+ return om.conf.Metadata.Retry.BackoffFunc(retries, om.conf.Metadata.Retry.Max)
+ } else {
+ return om.conf.Metadata.Retry.Backoff
+ }
+}
+
+func (om *offsetManager) fetchInitialOffset(topic string, partition int32, retries int) (int64, int32, string, error) {
+ broker, err := om.coordinator()
+ if err != nil {
+ if retries <= 0 {
+ return 0, 0, "", err
+ }
+ return om.fetchInitialOffset(topic, partition, retries-1)
+ }
+
+ partitions := map[string][]int32{topic: {partition}}
+ req := NewOffsetFetchRequest(om.conf.Version, om.group, partitions)
+ resp, err := broker.FetchOffset(req)
+ if err != nil {
+ if retries <= 0 {
+ return 0, 0, "", err
+ }
+ om.releaseCoordinator(broker)
+ return om.fetchInitialOffset(topic, partition, retries-1)
+ }
+
+ block := resp.GetBlock(topic, partition)
+ if block == nil {
+ return 0, 0, "", ErrIncompleteResponse
+ }
+
+ switch block.Err {
+ case ErrNoError:
+ return block.Offset, block.LeaderEpoch, block.Metadata, nil
+ case ErrNotCoordinatorForConsumer:
+ if retries <= 0 {
+ return 0, 0, "", block.Err
+ }
+ om.releaseCoordinator(broker)
+ return om.fetchInitialOffset(topic, partition, retries-1)
+ case ErrOffsetsLoadInProgress:
+ if retries <= 0 {
+ return 0, 0, "", block.Err
+ }
+ backoff := om.computeBackoff(retries)
+ select {
+ case <-om.closing:
+ return 0, 0, "", block.Err
+ case <-time.After(backoff):
+ }
+ return om.fetchInitialOffset(topic, partition, retries-1)
+ default:
+ return 0, 0, "", block.Err
+ }
+}
+
+func (om *offsetManager) coordinator() (*Broker, error) {
+ om.brokerLock.RLock()
+ broker := om.broker
+ om.brokerLock.RUnlock()
+
+ if broker != nil {
+ return broker, nil
+ }
+
+ om.brokerLock.Lock()
+ defer om.brokerLock.Unlock()
+
+ if broker := om.broker; broker != nil {
+ return broker, nil
+ }
+
+ if err := om.client.RefreshCoordinator(om.group); err != nil {
+ return nil, err
+ }
+
+ broker, err := om.client.Coordinator(om.group)
+ if err != nil {
+ return nil, err
+ }
+
+ om.broker = broker
+ return broker, nil
+}
+
+func (om *offsetManager) releaseCoordinator(b *Broker) {
+ om.brokerLock.Lock()
+ if om.broker == b {
+ om.broker = nil
+ }
+ om.brokerLock.Unlock()
+}
+
+func (om *offsetManager) mainLoop() {
+ defer om.ticker.Stop()
+ defer close(om.closed)
+
+ for {
+ select {
+ case <-om.ticker.C:
+ om.Commit()
+ case <-om.closing:
+ return
+ }
+ }
+}
+
+func (om *offsetManager) Commit() {
+ om.flushToBroker()
+ om.releasePOMs(false)
+}
+
+func (om *offsetManager) flushToBroker() {
+ broker, err := om.coordinator()
+ if err != nil {
+ om.handleError(err)
+ return
+ }
+
+ // Care needs to be taken to unlock this. Don't want to defer the unlock as this would
+ // cause the lock to be held while waiting for the broker to reply.
+ broker.lock.Lock()
+ req := om.constructRequest()
+ if req == nil {
+ broker.lock.Unlock()
+ return
+ }
+ resp, rp, err := sendOffsetCommit(broker, req)
+ broker.lock.Unlock()
+
+ if err != nil {
+ om.handleError(err)
+ om.releaseCoordinator(broker)
+ _ = broker.Close()
+ return
+ }
+
+ err = handleResponsePromise(req, resp, rp, nil)
+ if err != nil {
+ om.handleError(err)
+ om.releaseCoordinator(broker)
+ _ = broker.Close()
+ return
+ }
+
+ broker.handleThrottledResponse(resp)
+ om.handleResponse(broker, req, resp)
+}
+
+func sendOffsetCommit(coordinator *Broker, req *OffsetCommitRequest) (*OffsetCommitResponse, *responsePromise, error) {
+ resp := new(OffsetCommitResponse)
+
+ promise, err := coordinator.send(req, resp)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ return resp, promise, nil
+}
+
+func (om *offsetManager) constructRequest() *OffsetCommitRequest {
+ r := &OffsetCommitRequest{
+ Version: 1,
+ ConsumerGroup: om.group,
+ ConsumerID: om.memberID,
+ ConsumerGroupGeneration: om.generation,
+ }
+ // Version 1 adds timestamp and group membership information, as well as the commit timestamp.
+ //
+ // Version 2 adds retention time. It removes the commit timestamp added in version 1.
+ if om.conf.Version.IsAtLeast(V0_9_0_0) {
+ r.Version = 2
+ }
+ // Version 3 and 4 are the same as version 2.
+ if om.conf.Version.IsAtLeast(V0_11_0_0) {
+ r.Version = 3
+ }
+ if om.conf.Version.IsAtLeast(V2_0_0_0) {
+ r.Version = 4
+ }
+ // Version 5 removes the retention time, which is now controlled only by a broker configuration.
+ //
+ // Version 6 adds the leader epoch for fencing.
+ if om.conf.Version.IsAtLeast(V2_1_0_0) {
+ r.Version = 6
+ }
+ // version 7 adds a new field called groupInstanceId to indicate member identity across restarts.
+ if om.conf.Version.IsAtLeast(V2_3_0_0) {
+ r.Version = 7
+ r.GroupInstanceId = om.groupInstanceId
+ }
+
+ // commit timestamp was only briefly supported in V1 where we set it to
+ // ReceiveTime (-1) to tell the broker to set it to the time when the commit
+ // request was received
+ var commitTimestamp int64
+ if r.Version == 1 {
+ commitTimestamp = ReceiveTime
+ }
+
+ // request controlled retention was only supported from V2-V4 (it became
+ // broker-only after that) so if the user has set the config options then
+ // flow those through as retention time on the commit request.
+ if r.Version >= 2 && r.Version < 5 {
+ // Map Sarama's default of 0 to Kafka's default of -1
+ r.RetentionTime = -1
+ if om.conf.Consumer.Offsets.Retention > 0 {
+ r.RetentionTime = int64(om.conf.Consumer.Offsets.Retention / time.Millisecond)
+ }
+ }
+
+ om.pomsLock.RLock()
+ defer om.pomsLock.RUnlock()
+
+ for _, topicManagers := range om.poms {
+ for _, pom := range topicManagers {
+ pom.lock.Lock()
+ if pom.dirty {
+ r.AddBlockWithLeaderEpoch(pom.topic, pom.partition, pom.offset, pom.leaderEpoch, commitTimestamp, pom.metadata)
+ }
+ pom.lock.Unlock()
+ }
+ }
+
+ if len(r.blocks) > 0 {
+ return r
+ }
+
+ return nil
+}
+
+func (om *offsetManager) handleResponse(broker *Broker, req *OffsetCommitRequest, resp *OffsetCommitResponse) {
+ om.pomsLock.RLock()
+ defer om.pomsLock.RUnlock()
+
+ for _, topicManagers := range om.poms {
+ for _, pom := range topicManagers {
+ if req.blocks[pom.topic] == nil || req.blocks[pom.topic][pom.partition] == nil {
+ continue
+ }
+
+ var err KError
+ var ok bool
+
+ if resp.Errors[pom.topic] == nil {
+ pom.handleError(ErrIncompleteResponse)
+ continue
+ }
+ if err, ok = resp.Errors[pom.topic][pom.partition]; !ok {
+ pom.handleError(ErrIncompleteResponse)
+ continue
+ }
+
+ switch err {
+ case ErrNoError:
+ block := req.blocks[pom.topic][pom.partition]
+ pom.updateCommitted(block.offset, block.metadata)
+ case ErrNotLeaderForPartition, ErrLeaderNotAvailable,
+ ErrConsumerCoordinatorNotAvailable, ErrNotCoordinatorForConsumer:
+ // not a critical error, we just need to redispatch
+ om.releaseCoordinator(broker)
+ case ErrOffsetMetadataTooLarge, ErrInvalidCommitOffsetSize:
+ // nothing we can do about this, just tell the user and carry on
+ pom.handleError(err)
+ case ErrOffsetsLoadInProgress:
+ // nothing wrong but we didn't commit, we'll get it next time round
+ case ErrFencedInstancedId:
+ pom.handleError(err)
+ // TODO close the whole consumer for instance fenced....
+ om.tryCancelSession()
+ case ErrUnknownTopicOrPartition:
+ // let the user know *and* try redispatching - if topic-auto-create is
+ // enabled, redispatching should trigger a metadata req and create the
+ // topic; if not then re-dispatching won't help, but we've let the user
+ // know and it shouldn't hurt either (see https://github.com/IBM/sarama/issues/706)
+ fallthrough
+ default:
+ // dunno, tell the user and try redispatching
+ pom.handleError(err)
+ om.releaseCoordinator(broker)
+ }
+ }
+ }
+}
+
+func (om *offsetManager) handleError(err error) {
+ om.pomsLock.RLock()
+ defer om.pomsLock.RUnlock()
+
+ for _, topicManagers := range om.poms {
+ for _, pom := range topicManagers {
+ pom.handleError(err)
+ }
+ }
+}
+
+func (om *offsetManager) asyncClosePOMs() {
+ om.pomsLock.RLock()
+ defer om.pomsLock.RUnlock()
+
+ for _, topicManagers := range om.poms {
+ for _, pom := range topicManagers {
+ pom.AsyncClose()
+ }
+ }
+}
+
+// Releases/removes closed POMs once they are clean (or when forced)
+func (om *offsetManager) releasePOMs(force bool) (remaining int) {
+ om.pomsLock.Lock()
+ defer om.pomsLock.Unlock()
+
+ for topic, topicManagers := range om.poms {
+ for partition, pom := range topicManagers {
+ pom.lock.Lock()
+ releaseDue := pom.done && (force || !pom.dirty)
+ pom.lock.Unlock()
+
+ if releaseDue {
+ pom.release()
+
+ delete(om.poms[topic], partition)
+ if len(om.poms[topic]) == 0 {
+ delete(om.poms, topic)
+ }
+ }
+ }
+ remaining += len(om.poms[topic])
+ }
+ return
+}
+
+func (om *offsetManager) findPOM(topic string, partition int32) *partitionOffsetManager {
+ om.pomsLock.RLock()
+ defer om.pomsLock.RUnlock()
+
+ if partitions, ok := om.poms[topic]; ok {
+ if pom, ok := partitions[partition]; ok {
+ return pom
+ }
+ }
+ return nil
+}
+
+func (om *offsetManager) tryCancelSession() {
+ if om.sessionCanceler != nil {
+ om.sessionCanceler()
+ }
+}
+
+// Partition Offset Manager
+
+// PartitionOffsetManager uses Kafka to store and fetch consumed partition offsets. You MUST call Close()
+// on a partition offset manager to avoid leaks, it will not be garbage-collected automatically when it passes
+// out of scope.
+type PartitionOffsetManager interface {
+ // NextOffset returns the next offset that should be consumed for the managed
+ // partition, accompanied by metadata which can be used to reconstruct the state
+ // of the partition consumer when it resumes. NextOffset() will return
+ // `config.Consumer.Offsets.Initial` and an empty metadata string if no offset
+ // was committed for this partition yet.
+ NextOffset() (int64, string)
+
+ // MarkOffset marks the provided offset, alongside a metadata string
+ // that represents the state of the partition consumer at that point in time. The
+ // metadata string can be used by another consumer to restore that state, so it
+ // can resume consumption.
+ //
+ // To follow upstream conventions, you are expected to mark the offset of the
+ // next message to read, not the last message read. Thus, when calling `MarkOffset`
+ // you should typically add one to the offset of the last consumed message.
+ //
+ // Note: calling MarkOffset does not necessarily commit the offset to the backend
+ // store immediately for efficiency reasons, and it may never be committed if
+ // your application crashes. This means that you may end up processing the same
+ // message twice, and your processing should ideally be idempotent.
+ MarkOffset(offset int64, metadata string)
+
+ // ResetOffset resets to the provided offset, alongside a metadata string that
+ // represents the state of the partition consumer at that point in time. Reset
+ // acts as a counterpart to MarkOffset, the difference being that it allows to
+ // reset an offset to an earlier or smaller value, where MarkOffset only
+ // allows incrementing the offset. cf MarkOffset for more details.
+ ResetOffset(offset int64, metadata string)
+
+ // Errors returns a read channel of errors that occur during offset management, if
+ // enabled. By default, errors are logged and not returned over this channel. If
+ // you want to implement any custom error handling, set your config's
+ // Consumer.Return.Errors setting to true, and read from this channel.
+ Errors() <-chan *ConsumerError
+
+ // AsyncClose initiates a shutdown of the PartitionOffsetManager. This method will
+ // return immediately, after which you should wait until the 'errors' channel has
+ // been drained and closed. It is required to call this function, or Close before
+ // a consumer object passes out of scope, as it will otherwise leak memory. You
+ // must call this before calling Close on the underlying client.
+ AsyncClose()
+
+ // Close stops the PartitionOffsetManager from managing offsets. It is required to
+ // call this function (or AsyncClose) before a PartitionOffsetManager object
+ // passes out of scope, as it will otherwise leak memory. You must call this
+ // before calling Close on the underlying client.
+ Close() error
+}
+
+type partitionOffsetManager struct {
+ parent *offsetManager
+ topic string
+ partition int32
+ leaderEpoch int32
+
+ lock sync.Mutex
+ offset int64
+ metadata string
+ dirty bool
+ done bool
+
+ releaseOnce sync.Once
+ errors chan *ConsumerError
+}
+
+func (om *offsetManager) newPartitionOffsetManager(topic string, partition int32) (*partitionOffsetManager, error) {
+ offset, leaderEpoch, metadata, err := om.fetchInitialOffset(topic, partition, om.conf.Metadata.Retry.Max)
+ if err != nil {
+ return nil, err
+ }
+
+ return &partitionOffsetManager{
+ parent: om,
+ topic: topic,
+ partition: partition,
+ leaderEpoch: leaderEpoch,
+ errors: make(chan *ConsumerError, om.conf.ChannelBufferSize),
+ offset: offset,
+ metadata: metadata,
+ }, nil
+}
+
+func (pom *partitionOffsetManager) Errors() <-chan *ConsumerError {
+ return pom.errors
+}
+
+func (pom *partitionOffsetManager) MarkOffset(offset int64, metadata string) {
+ pom.lock.Lock()
+ defer pom.lock.Unlock()
+
+ if offset > pom.offset {
+ pom.offset = offset
+ pom.metadata = metadata
+ pom.dirty = true
+ }
+}
+
+func (pom *partitionOffsetManager) ResetOffset(offset int64, metadata string) {
+ pom.lock.Lock()
+ defer pom.lock.Unlock()
+
+ if offset <= pom.offset {
+ pom.offset = offset
+ pom.metadata = metadata
+ pom.dirty = true
+ }
+}
+
+func (pom *partitionOffsetManager) updateCommitted(offset int64, metadata string) {
+ pom.lock.Lock()
+ defer pom.lock.Unlock()
+
+ if pom.offset == offset && pom.metadata == metadata {
+ pom.dirty = false
+ }
+}
+
+func (pom *partitionOffsetManager) NextOffset() (int64, string) {
+ pom.lock.Lock()
+ defer pom.lock.Unlock()
+
+ if pom.offset >= 0 {
+ return pom.offset, pom.metadata
+ }
+
+ return pom.parent.conf.Consumer.Offsets.Initial, ""
+}
+
+func (pom *partitionOffsetManager) AsyncClose() {
+ pom.lock.Lock()
+ pom.done = true
+ pom.lock.Unlock()
+}
+
+func (pom *partitionOffsetManager) Close() error {
+ pom.AsyncClose()
+
+ var errors ConsumerErrors
+ for err := range pom.errors {
+ errors = append(errors, err)
+ }
+
+ if len(errors) > 0 {
+ return errors
+ }
+ return nil
+}
+
+func (pom *partitionOffsetManager) handleError(err error) {
+ cErr := &ConsumerError{
+ Topic: pom.topic,
+ Partition: pom.partition,
+ Err: err,
+ }
+
+ if pom.parent.conf.Consumer.Return.Errors {
+ pom.errors <- cErr
+ } else {
+ Logger.Println(cErr)
+ }
+}
+
+func (pom *partitionOffsetManager) release() {
+ pom.releaseOnce.Do(func() {
+ close(pom.errors)
+ })
+}
diff --git a/vendor/github.com/IBM/sarama/offset_request.go b/vendor/github.com/IBM/sarama/offset_request.go
new file mode 100644
index 0000000..01fbb33
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/offset_request.go
@@ -0,0 +1,236 @@
+package sarama
+
+type offsetRequestBlock struct {
+ // currentLeaderEpoch contains the current leader epoch (used in version 4+).
+ currentLeaderEpoch int32
+ // timestamp contains the current timestamp.
+ timestamp int64
+ // maxNumOffsets contains the maximum number of offsets to report.
+ maxNumOffsets int32 // Only used in version 0
+}
+
+func (b *offsetRequestBlock) encode(pe packetEncoder, version int16) error {
+ if version >= 4 {
+ pe.putInt32(b.currentLeaderEpoch)
+ }
+
+ pe.putInt64(b.timestamp)
+
+ if version == 0 {
+ pe.putInt32(b.maxNumOffsets)
+ }
+
+ return nil
+}
+
+func (b *offsetRequestBlock) decode(pd packetDecoder, version int16) (err error) {
+ b.currentLeaderEpoch = -1
+ if version >= 4 {
+ if b.currentLeaderEpoch, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+
+ if b.timestamp, err = pd.getInt64(); err != nil {
+ return err
+ }
+
+ if version == 0 {
+ if b.maxNumOffsets, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+type OffsetRequest struct {
+ Version int16
+ IsolationLevel IsolationLevel
+ replicaID int32
+ isReplicaIDSet bool
+ blocks map[string]map[int32]*offsetRequestBlock
+}
+
+func NewOffsetRequest(version KafkaVersion) *OffsetRequest {
+ request := &OffsetRequest{}
+ if version.IsAtLeast(V2_2_0_0) {
+ // Version 5 adds a new error code, OFFSET_NOT_AVAILABLE.
+ request.Version = 5
+ } else if version.IsAtLeast(V2_1_0_0) {
+ // Version 4 adds the current leader epoch, which is used for fencing.
+ request.Version = 4
+ } else if version.IsAtLeast(V2_0_0_0) {
+ // Version 3 is the same as version 2.
+ request.Version = 3
+ } else if version.IsAtLeast(V0_11_0_0) {
+ // Version 2 adds the isolation level, which is used for transactional reads.
+ request.Version = 2
+ } else if version.IsAtLeast(V0_10_1_0) {
+ // Version 1 removes MaxNumOffsets. From this version forward, only a single
+ // offset can be returned.
+ request.Version = 1
+ }
+ return request
+}
+
+func (r *OffsetRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *OffsetRequest) encode(pe packetEncoder) error {
+ if r.isReplicaIDSet {
+ pe.putInt32(r.replicaID)
+ } else {
+ // default replica ID is always -1 for clients
+ pe.putInt32(-1)
+ }
+
+ if r.Version >= 2 {
+ pe.putBool(r.IsolationLevel == ReadCommitted)
+ }
+
+ err := pe.putArrayLength(len(r.blocks))
+ if err != nil {
+ return err
+ }
+ for topic, partitions := range r.blocks {
+ err = pe.putString(topic)
+ if err != nil {
+ return err
+ }
+ err = pe.putArrayLength(len(partitions))
+ if err != nil {
+ return err
+ }
+ for partition, block := range partitions {
+ pe.putInt32(partition)
+ if err = block.encode(pe, r.Version); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func (r *OffsetRequest) decode(pd packetDecoder, version int16) error {
+ r.Version = version
+
+ replicaID, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ if replicaID >= 0 {
+ r.SetReplicaID(replicaID)
+ }
+
+ if r.Version >= 2 {
+ tmp, err := pd.getBool()
+ if err != nil {
+ return err
+ }
+
+ r.IsolationLevel = ReadUncommitted
+ if tmp {
+ r.IsolationLevel = ReadCommitted
+ }
+ }
+
+ blockCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if blockCount == 0 {
+ return nil
+ }
+ r.blocks = make(map[string]map[int32]*offsetRequestBlock)
+ for i := 0; i < blockCount; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ partitionCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ r.blocks[topic] = make(map[int32]*offsetRequestBlock)
+ for j := 0; j < partitionCount; j++ {
+ partition, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ block := &offsetRequestBlock{}
+ if err := block.decode(pd, version); err != nil {
+ return err
+ }
+ r.blocks[topic][partition] = block
+ }
+ }
+ return nil
+}
+
+func (r *OffsetRequest) key() int16 {
+ return apiKeyListOffsets
+}
+
+func (r *OffsetRequest) version() int16 {
+ return r.Version
+}
+
+func (r *OffsetRequest) headerVersion() int16 {
+ return 1
+}
+
+func (r *OffsetRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 5
+}
+
+func (r *OffsetRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 5:
+ return V2_2_0_0
+ case 4:
+ return V2_1_0_0
+ case 3:
+ return V2_0_0_0
+ case 2:
+ return V0_11_0_0
+ case 1:
+ return V0_10_1_0
+ case 0:
+ return V0_8_2_0
+ default:
+ return V2_0_0_0
+ }
+}
+
+func (r *OffsetRequest) SetReplicaID(id int32) {
+ r.replicaID = id
+ r.isReplicaIDSet = true
+}
+
+func (r *OffsetRequest) ReplicaID() int32 {
+ if r.isReplicaIDSet {
+ return r.replicaID
+ }
+ return -1
+}
+
+func (r *OffsetRequest) AddBlock(topic string, partitionID int32, timestamp int64, maxOffsets int32) {
+ if r.blocks == nil {
+ r.blocks = make(map[string]map[int32]*offsetRequestBlock)
+ }
+
+ if r.blocks[topic] == nil {
+ r.blocks[topic] = make(map[int32]*offsetRequestBlock)
+ }
+
+ tmp := new(offsetRequestBlock)
+ tmp.currentLeaderEpoch = -1
+ tmp.timestamp = timestamp
+ if r.Version == 0 {
+ tmp.maxNumOffsets = maxOffsets
+ }
+
+ r.blocks[topic][partitionID] = tmp
+}
diff --git a/vendor/github.com/IBM/sarama/offset_response.go b/vendor/github.com/IBM/sarama/offset_response.go
new file mode 100644
index 0000000..d57c24f
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/offset_response.go
@@ -0,0 +1,230 @@
+package sarama
+
+import "time"
+
+type OffsetResponseBlock struct {
+ Err KError
+ // Offsets contains the result offsets (for V0/V1 compatibility)
+ Offsets []int64 // Version 0
+ // Timestamp contains the timestamp associated with the returned offset.
+ Timestamp int64 // Version 1
+ // Offset contains the returned offset.
+ Offset int64 // Version 1
+ // LeaderEpoch contains the current leader epoch of the partition.
+ LeaderEpoch int32
+}
+
+func (b *OffsetResponseBlock) decode(pd packetDecoder, version int16) (err error) {
+ b.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if version == 0 {
+ b.Offsets, err = pd.getInt64Array()
+ return err
+ }
+
+ if version >= 1 {
+ b.Timestamp, err = pd.getInt64()
+ if err != nil {
+ return err
+ }
+
+ b.Offset, err = pd.getInt64()
+ if err != nil {
+ return err
+ }
+
+ // For backwards compatibility put the offset in the offsets array too
+ b.Offsets = []int64{b.Offset}
+ }
+
+ if version >= 4 {
+ if b.LeaderEpoch, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (b *OffsetResponseBlock) encode(pe packetEncoder, version int16) (err error) {
+ pe.putKError(b.Err)
+
+ if version == 0 {
+ return pe.putInt64Array(b.Offsets)
+ }
+
+ if version >= 1 {
+ pe.putInt64(b.Timestamp)
+ pe.putInt64(b.Offset)
+ }
+
+ if version >= 4 {
+ pe.putInt32(b.LeaderEpoch)
+ }
+
+ return nil
+}
+
+type OffsetResponse struct {
+ Version int16
+ ThrottleTimeMs int32
+ Blocks map[string]map[int32]*OffsetResponseBlock
+}
+
+func (r *OffsetResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *OffsetResponse) decode(pd packetDecoder, version int16) (err error) {
+ if version >= 2 {
+ r.ThrottleTimeMs, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+ }
+
+ numTopics, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Blocks = make(map[string]map[int32]*OffsetResponseBlock, numTopics)
+ for i := 0; i < numTopics; i++ {
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ numBlocks, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Blocks[name] = make(map[int32]*OffsetResponseBlock, numBlocks)
+
+ for j := 0; j < numBlocks; j++ {
+ id, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ block := new(OffsetResponseBlock)
+ err = block.decode(pd, version)
+ if err != nil {
+ return err
+ }
+ r.Blocks[name][id] = block
+ }
+ }
+
+ return nil
+}
+
+func (r *OffsetResponse) GetBlock(topic string, partition int32) *OffsetResponseBlock {
+ if r.Blocks == nil {
+ return nil
+ }
+
+ if r.Blocks[topic] == nil {
+ return nil
+ }
+
+ return r.Blocks[topic][partition]
+}
+
+/*
+// [0 0 0 1 ntopics
+0 8 109 121 95 116 111 112 105 99 topic
+0 0 0 1 npartitions
+0 0 0 0 id
+0 0
+
+0 0 0 1 0 0 0 0
+0 1 1 1 0 0 0 1
+0 8 109 121 95 116 111 112
+105 99 0 0 0 1 0 0
+0 0 0 0 0 0 0 1
+0 0 0 0 0 1 1 1] <nil>
+*/
+func (r *OffsetResponse) encode(pe packetEncoder) (err error) {
+ if r.Version >= 2 {
+ pe.putInt32(r.ThrottleTimeMs)
+ }
+
+ if err = pe.putArrayLength(len(r.Blocks)); err != nil {
+ return err
+ }
+
+ for topic, partitions := range r.Blocks {
+ if err = pe.putString(topic); err != nil {
+ return err
+ }
+ if err = pe.putArrayLength(len(partitions)); err != nil {
+ return err
+ }
+ for partition, block := range partitions {
+ pe.putInt32(partition)
+ if err = block.encode(pe, r.version()); err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func (r *OffsetResponse) key() int16 {
+ return apiKeyListOffsets
+}
+
+func (r *OffsetResponse) version() int16 {
+ return r.Version
+}
+
+func (r *OffsetResponse) headerVersion() int16 {
+ return 0
+}
+
+func (r *OffsetResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 5
+}
+
+func (r *OffsetResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 5:
+ return V2_2_0_0
+ case 4:
+ return V2_1_0_0
+ case 3:
+ return V2_0_0_0
+ case 2:
+ return V0_11_0_0
+ case 1:
+ return V0_10_1_0
+ case 0:
+ return V0_8_2_0
+ default:
+ return V2_0_0_0
+ }
+}
+
+func (r *OffsetResponse) throttleTime() time.Duration {
+ return time.Duration(r.ThrottleTimeMs) * time.Millisecond
+}
+
+// testing API
+
+func (r *OffsetResponse) AddTopicPartition(topic string, partition int32, offset int64) {
+ if r.Blocks == nil {
+ r.Blocks = make(map[string]map[int32]*OffsetResponseBlock)
+ }
+ byTopic, ok := r.Blocks[topic]
+ if !ok {
+ byTopic = make(map[int32]*OffsetResponseBlock)
+ r.Blocks[topic] = byTopic
+ }
+ byTopic[partition] = &OffsetResponseBlock{Offsets: []int64{offset}, Offset: offset}
+}
diff --git a/vendor/github.com/IBM/sarama/packet_decoder.go b/vendor/github.com/IBM/sarama/packet_decoder.go
new file mode 100644
index 0000000..241c6c6
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/packet_decoder.go
@@ -0,0 +1,79 @@
+package sarama
+
+import (
+ "time"
+
+ "github.com/rcrowley/go-metrics"
+)
+
+type taggedFieldDecoderFunc func(pd packetDecoder) error
+type taggedFieldDecoders map[uint64]taggedFieldDecoderFunc
+
+// PacketDecoder is the interface providing helpers for reading with Kafka's encoding rules.
+// Types implementing Decoder only need to worry about calling methods like GetString,
+// not about how a string is represented in Kafka.
+type packetDecoder interface {
+ // Primitives
+ getInt8() (int8, error)
+ getInt16() (int16, error)
+ getInt32() (int32, error)
+ getInt64() (int64, error)
+ getVarint() (int64, error)
+ getUVarint() (uint64, error)
+ getFloat64() (float64, error)
+ getArrayLength() (int, error)
+ getBool() (bool, error)
+ getKError() (KError, error)
+ getDurationMs() (time.Duration, error)
+ getEmptyTaggedFieldArray() (int, error)
+ getTaggedFieldArray(taggedFieldDecoders) error
+
+ // Collections
+ getBytes() ([]byte, error)
+ getVarintBytes() ([]byte, error)
+ getRawBytes(length int) ([]byte, error)
+ getString() (string, error)
+ getNullableString() (*string, error)
+ getInt32Array() ([]int32, error)
+ getInt64Array() ([]int64, error)
+ getStringArray() ([]string, error)
+
+ // Subsets
+ remaining() int
+ getSubset(length int) (packetDecoder, error)
+ peek(offset, length int) (packetDecoder, error) // similar to getSubset, but it doesn't advance the offset
+ peekInt8(offset int) (int8, error) // similar to peek, but just one byte
+
+ // Stacks, see PushDecoder
+ push(in pushDecoder) error
+ pop() error
+
+ // To record metrics when provided
+ metricRegistry() metrics.Registry
+}
+
+// PushDecoder is the interface for decoding fields like CRCs and lengths where the validity
+// of the field depends on what is after it in the packet. Start them with PacketDecoder.Push() where
+// the actual value is located in the packet, then PacketDecoder.Pop() them when all the bytes they
+// depend upon have been decoded.
+type pushDecoder interface {
+ // Saves the offset into the input buffer as the location to actually read the calculated value when able.
+ saveOffset(in int)
+
+ // Returns the length of data to reserve for the input of this encoder (e.g. 4 bytes for a CRC32).
+ reserveLength() int
+
+ // Indicates that all required data is now available to calculate and check the field.
+ // SaveOffset is guaranteed to have been called first. The implementation should read ReserveLength() bytes
+ // of data from the saved offset, and verify it based on the data between the saved offset and curOffset.
+ check(curOffset int, buf []byte) error
+}
+
+// dynamicPushDecoder extends the interface of pushDecoder for uses cases where the length of the
+// fields itself is unknown until its value was decoded (for instance varint encoded length
+// fields).
+// During push, dynamicPushDecoder.decode() method will be called instead of reserveLength()
+type dynamicPushDecoder interface {
+ pushDecoder
+ decoder
+}
diff --git a/vendor/github.com/IBM/sarama/packet_encoder.go b/vendor/github.com/IBM/sarama/packet_encoder.go
new file mode 100644
index 0000000..674a550
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/packet_encoder.go
@@ -0,0 +1,75 @@
+package sarama
+
+import (
+ "time"
+
+ "github.com/rcrowley/go-metrics"
+)
+
+// PacketEncoder is the interface providing helpers for writing with Kafka's encoding rules.
+// Types implementing Encoder only need to worry about calling methods like PutString,
+// not about how a string is represented in Kafka.
+type packetEncoder interface {
+ // Primitives
+ putInt8(in int8)
+ putInt16(in int16)
+ putInt32(in int32)
+ putInt64(in int64)
+ putVarint(in int64)
+ putUVarint(in uint64)
+ putFloat64(in float64)
+ putArrayLength(in int) error
+ putBool(in bool)
+ putKError(in KError)
+ putDurationMs(in time.Duration)
+
+ // Collections
+ putBytes(in []byte) error
+ putVarintBytes(in []byte) error
+ putRawBytes(in []byte) error
+ putString(in string) error
+ putNullableString(in *string) error
+ putStringArray(in []string) error
+ putInt32Array(in []int32) error
+ putInt64Array(in []int64) error
+ putNullableInt32Array(in []int32) error
+ putEmptyTaggedFieldArray()
+
+ // Provide the current offset to record the batch size metric
+ offset() int
+
+ // Stacks, see PushEncoder
+ push(in pushEncoder)
+ pop() error
+
+ // To record metrics when provided
+ metricRegistry() metrics.Registry
+}
+
+// PushEncoder is the interface for encoding fields like CRCs and lengths where the value
+// of the field depends on what is encoded after it in the packet. Start them with PacketEncoder.Push() where
+// the actual value is located in the packet, then PacketEncoder.Pop() them when all the bytes they
+// depend upon have been written.
+type pushEncoder interface {
+ // Saves the offset into the input buffer as the location to actually write the calculated value when able.
+ saveOffset(in int)
+
+ // Returns the length of data to reserve for the output of this encoder (eg 4 bytes for a CRC32).
+ reserveLength() int
+
+ // Indicates that all required data is now available to calculate and write the field.
+ // SaveOffset is guaranteed to have been called first. The implementation should write ReserveLength() bytes
+ // of data to the saved offset, based on the data between the saved offset and curOffset.
+ run(curOffset int, buf []byte) error
+}
+
+// dynamicPushEncoder extends the interface of pushEncoder for uses cases where the length of the
+// fields itself is unknown until its value was computed (for instance varint encoded length
+// fields).
+type dynamicPushEncoder interface {
+ pushEncoder
+
+ // Called during pop() to adjust the length of the field.
+ // It should return the difference in bytes between the last computed length and current length.
+ adjustLength(currOffset int) int
+}
diff --git a/vendor/github.com/IBM/sarama/partitioner.go b/vendor/github.com/IBM/sarama/partitioner.go
new file mode 100644
index 0000000..50a345a
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/partitioner.go
@@ -0,0 +1,248 @@
+package sarama
+
+import (
+ "hash"
+ "hash/crc32"
+ "hash/fnv"
+ "math/rand"
+ "time"
+)
+
+// Partitioner is anything that, given a Kafka message and a number of partitions indexed [0...numPartitions-1],
+// decides to which partition to send the message. RandomPartitioner, RoundRobinPartitioner and HashPartitioner are provided
+// as simple default implementations.
+type Partitioner interface {
+ // Partition takes a message and partition count and chooses a partition
+ Partition(message *ProducerMessage, numPartitions int32) (int32, error)
+
+ // RequiresConsistency indicates to the user of the partitioner whether the
+ // mapping of key->partition is consistent or not. Specifically, if a
+ // partitioner requires consistency then it must be allowed to choose from all
+ // partitions (even ones known to be unavailable), and its choice must be
+ // respected by the caller. The obvious example is the HashPartitioner.
+ RequiresConsistency() bool
+}
+
+// DynamicConsistencyPartitioner can optionally be implemented by Partitioners
+// in order to allow more flexibility than is originally allowed by the
+// RequiresConsistency method in the Partitioner interface. This allows
+// partitioners to require consistency sometimes, but not all times. It's useful
+// for, e.g., the HashPartitioner, which does not require consistency if the
+// message key is nil.
+type DynamicConsistencyPartitioner interface {
+ Partitioner
+
+ // MessageRequiresConsistency is similar to Partitioner.RequiresConsistency,
+ // but takes in the message being partitioned so that the partitioner can
+ // make a per-message determination.
+ MessageRequiresConsistency(message *ProducerMessage) bool
+}
+
+// PartitionerConstructor is the type for a function capable of constructing new Partitioners.
+type PartitionerConstructor func(topic string) Partitioner
+
+type manualPartitioner struct{}
+
+// HashPartitionerOption lets you modify default values of the partitioner
+type HashPartitionerOption func(*hashPartitioner)
+
+// WithAbsFirst means that the partitioner handles absolute values
+// in the same way as the reference Java implementation
+func WithAbsFirst() HashPartitionerOption {
+ return func(hp *hashPartitioner) {
+ hp.referenceAbs = true
+ }
+}
+
+// WithHashUnsigned means the partitioner treats the hashed value as unsigned when
+// partitioning. This is intended to be combined with the crc32 hash algorithm to
+// be compatible with librdkafka's implementation
+func WithHashUnsigned() HashPartitionerOption {
+ return func(hp *hashPartitioner) {
+ hp.hashUnsigned = true
+ }
+}
+
+// WithCustomHashFunction lets you specify what hash function to use for the partitioning
+func WithCustomHashFunction(hasher func() hash.Hash32) HashPartitionerOption {
+ return func(hp *hashPartitioner) {
+ hp.hasher = hasher()
+ }
+}
+
+// WithCustomFallbackPartitioner lets you specify what HashPartitioner should be used in case a Distribution Key is empty
+func WithCustomFallbackPartitioner(randomHP Partitioner) HashPartitionerOption {
+ return func(hp *hashPartitioner) {
+ hp.random = randomHP
+ }
+}
+
+// NewManualPartitioner returns a Partitioner which uses the partition manually set in the provided
+// ProducerMessage's Partition field as the partition to produce to.
+func NewManualPartitioner(topic string) Partitioner {
+ return new(manualPartitioner)
+}
+
+func (p *manualPartitioner) Partition(message *ProducerMessage, numPartitions int32) (int32, error) {
+ return message.Partition, nil
+}
+
+func (p *manualPartitioner) RequiresConsistency() bool {
+ return true
+}
+
+type randomPartitioner struct {
+ generator *rand.Rand
+}
+
+// NewRandomPartitioner returns a Partitioner which chooses a random partition each time.
+func NewRandomPartitioner(topic string) Partitioner {
+ p := new(randomPartitioner)
+ p.generator = rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
+ return p
+}
+
+func (p *randomPartitioner) Partition(message *ProducerMessage, numPartitions int32) (int32, error) {
+ return int32(p.generator.Intn(int(numPartitions))), nil
+}
+
+func (p *randomPartitioner) RequiresConsistency() bool {
+ return false
+}
+
+type roundRobinPartitioner struct {
+ partition int32
+}
+
+// NewRoundRobinPartitioner returns a Partitioner which walks through the available partitions one at a time.
+func NewRoundRobinPartitioner(topic string) Partitioner {
+ return &roundRobinPartitioner{}
+}
+
+func (p *roundRobinPartitioner) Partition(message *ProducerMessage, numPartitions int32) (int32, error) {
+ if p.partition >= numPartitions {
+ p.partition = 0
+ }
+ ret := p.partition
+ p.partition++
+ return ret, nil
+}
+
+func (p *roundRobinPartitioner) RequiresConsistency() bool {
+ return false
+}
+
+type hashPartitioner struct {
+ random Partitioner
+ hasher hash.Hash32
+ referenceAbs bool
+ hashUnsigned bool
+}
+
+// NewCustomHashPartitioner is a wrapper around NewHashPartitioner, allowing the use of custom hasher.
+// The argument is a function providing the instance, implementing the hash.Hash32 interface. This is to ensure that
+// each partition dispatcher gets its own hasher, to avoid concurrency issues by sharing an instance.
+func NewCustomHashPartitioner(hasher func() hash.Hash32) PartitionerConstructor {
+ return func(topic string) Partitioner {
+ p := new(hashPartitioner)
+ p.random = NewRandomPartitioner(topic)
+ p.hasher = hasher()
+ p.referenceAbs = false
+ p.hashUnsigned = false
+ return p
+ }
+}
+
+// NewCustomPartitioner creates a default Partitioner but lets you specify the behavior of each component via options
+func NewCustomPartitioner(options ...HashPartitionerOption) PartitionerConstructor {
+ return func(topic string) Partitioner {
+ p := new(hashPartitioner)
+ p.random = NewRandomPartitioner(topic)
+ p.hasher = fnv.New32a()
+ p.referenceAbs = false
+ p.hashUnsigned = false
+ for _, option := range options {
+ option(p)
+ }
+ return p
+ }
+}
+
+// NewHashPartitioner returns a Partitioner which behaves as follows. If the message's key is nil then a
+// random partition is chosen. Otherwise the FNV-1a hash of the encoded bytes of the message key is used,
+// modulus the number of partitions. This ensures that messages with the same key always end up on the
+// same partition.
+func NewHashPartitioner(topic string) Partitioner {
+ p := new(hashPartitioner)
+ p.random = NewRandomPartitioner(topic)
+ p.hasher = fnv.New32a()
+ p.referenceAbs = false
+ p.hashUnsigned = false
+ return p
+}
+
+// NewReferenceHashPartitioner is like NewHashPartitioner except that it handles absolute values
+// in the same way as the reference Java implementation. NewHashPartitioner was supposed to do
+// that but it had a mistake and now there are people depending on both behaviors. This will
+// all go away on the next major version bump.
+func NewReferenceHashPartitioner(topic string) Partitioner {
+ p := new(hashPartitioner)
+ p.random = NewRandomPartitioner(topic)
+ p.hasher = fnv.New32a()
+ p.referenceAbs = true
+ p.hashUnsigned = false
+ return p
+}
+
+// NewConsistentCRCHashPartitioner is like NewHashPartitioner execpt that it uses the *unsigned* crc32 hash
+// of the encoded bytes of the message key modulus the number of partitions. This is compatible with
+// librdkafka's `consistent_random` partitioner
+func NewConsistentCRCHashPartitioner(topic string) Partitioner {
+ p := new(hashPartitioner)
+ p.random = NewRandomPartitioner(topic)
+ p.hasher = crc32.NewIEEE()
+ p.referenceAbs = false
+ p.hashUnsigned = true
+ return p
+}
+
+func (p *hashPartitioner) Partition(message *ProducerMessage, numPartitions int32) (int32, error) {
+ if message.Key == nil {
+ return p.random.Partition(message, numPartitions)
+ }
+ bytes, err := message.Key.Encode()
+ if err != nil {
+ return -1, err
+ }
+ p.hasher.Reset()
+ _, err = p.hasher.Write(bytes)
+ if err != nil {
+ return -1, err
+ }
+ var partition int32
+ // Turns out we were doing our absolute value in a subtly different way from the upstream
+ // implementation, but now we need to maintain backwards compat for people who started using
+ // the old version; if referenceAbs is set we are compatible with the reference java client
+ // but not past Sarama versions
+ if p.referenceAbs {
+ partition = (int32(p.hasher.Sum32()) & 0x7fffffff) % numPartitions
+ } else if p.hashUnsigned {
+ // librdkafka treats the hashed value as unsigned. If `hashUnsigned` is set we are compatible
+ // with librdkafka's `consistent` partitioning but not past Sarama versions
+ partition = int32(p.hasher.Sum32() % uint32(numPartitions))
+ } else {
+ partition = int32(p.hasher.Sum32()) % numPartitions
+ if partition < 0 {
+ partition = -partition
+ }
+ }
+ return partition, nil
+}
+
+func (p *hashPartitioner) RequiresConsistency() bool {
+ return true
+}
+
+func (p *hashPartitioner) MessageRequiresConsistency(message *ProducerMessage) bool {
+ return message.Key != nil
+}
diff --git a/vendor/github.com/IBM/sarama/prep_encoder.go b/vendor/github.com/IBM/sarama/prep_encoder.go
new file mode 100644
index 0000000..d30f6e5
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/prep_encoder.go
@@ -0,0 +1,258 @@
+package sarama
+
+import (
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "math"
+ "time"
+
+ "github.com/rcrowley/go-metrics"
+)
+
+type prepEncoder struct {
+ stack []pushEncoder
+ length int
+}
+
+type prepFlexibleEncoder struct {
+ *prepEncoder
+}
+
+// primitives
+
+func (pe *prepEncoder) putInt8(in int8) {
+ pe.length++
+}
+
+func (pe *prepEncoder) putInt16(in int16) {
+ pe.length += 2
+}
+
+func (pe *prepEncoder) putInt32(in int32) {
+ pe.length += 4
+}
+
+func (pe *prepEncoder) putInt64(in int64) {
+ pe.length += 8
+}
+
+func (pe *prepEncoder) putVarint(in int64) {
+ var buf [binary.MaxVarintLen64]byte
+ pe.length += binary.PutVarint(buf[:], in)
+}
+
+func (pe *prepEncoder) putUVarint(in uint64) {
+ var buf [binary.MaxVarintLen64]byte
+ pe.length += binary.PutUvarint(buf[:], in)
+}
+
+func (pe *prepEncoder) putFloat64(in float64) {
+ pe.length += 8
+}
+
+func (pe *prepEncoder) putArrayLength(in int) error {
+ if in > math.MaxInt32 {
+ return PacketEncodingError{fmt.Sprintf("array too long (%d)", in)}
+ }
+ pe.length += 4
+ return nil
+}
+
+func (pe *prepEncoder) putBool(in bool) {
+ pe.length++
+}
+
+func (pe *prepEncoder) putKError(in KError) {
+ pe.length += 2
+}
+
+func (pe *prepEncoder) putDurationMs(in time.Duration) {
+ pe.length += 4
+}
+
+// arrays
+
+func (pe *prepEncoder) putBytes(in []byte) error {
+ pe.length += 4
+ if in == nil {
+ return nil
+ }
+ return pe.putRawBytes(in)
+}
+
+func (pe *prepEncoder) putVarintBytes(in []byte) error {
+ if in == nil {
+ pe.putVarint(-1)
+ return nil
+ }
+ pe.putVarint(int64(len(in)))
+ return pe.putRawBytes(in)
+}
+
+func (pe *prepEncoder) putRawBytes(in []byte) error {
+ if len(in) > math.MaxInt32 {
+ return PacketEncodingError{fmt.Sprintf("byteslice too long (%d)", len(in))}
+ }
+ pe.length += len(in)
+ return nil
+}
+
+func (pe *prepEncoder) putNullableString(in *string) error {
+ if in == nil {
+ pe.length += 2
+ return nil
+ }
+ return pe.putString(*in)
+}
+
+func (pe *prepEncoder) putString(in string) error {
+ pe.length += 2
+ if len(in) > math.MaxInt16 {
+ return PacketEncodingError{fmt.Sprintf("string too long (%d)", len(in))}
+ }
+ pe.length += len(in)
+ return nil
+}
+
+func (pe *prepEncoder) putStringArray(in []string) error {
+ err := pe.putArrayLength(len(in))
+ if err != nil {
+ return err
+ }
+
+ for _, str := range in {
+ if err := pe.putString(str); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (pe *prepEncoder) putInt32Array(in []int32) error {
+ err := pe.putArrayLength(len(in))
+ if err != nil {
+ return err
+ }
+ pe.length += 4 * len(in)
+ return nil
+}
+
+func (pe *prepEncoder) putNullableInt32Array(in []int32) error {
+ if in == nil {
+ pe.length += 4
+ return nil
+ }
+ err := pe.putArrayLength(len(in))
+ if err != nil {
+ return err
+ }
+ pe.length += 4 * len(in)
+ return nil
+}
+
+func (pe *prepEncoder) putInt64Array(in []int64) error {
+ err := pe.putArrayLength(len(in))
+ if err != nil {
+ return err
+ }
+ pe.length += 8 * len(in)
+ return nil
+}
+
+func (pe *prepEncoder) putEmptyTaggedFieldArray() {
+}
+
+func (pe *prepEncoder) offset() int {
+ return pe.length
+}
+
+// stackable
+
+func (pe *prepEncoder) push(in pushEncoder) {
+ in.saveOffset(pe.length)
+ pe.length += in.reserveLength()
+ pe.stack = append(pe.stack, in)
+}
+
+func (pe *prepEncoder) pop() error {
+ in := pe.stack[len(pe.stack)-1]
+ pe.stack = pe.stack[:len(pe.stack)-1]
+ if dpe, ok := in.(dynamicPushEncoder); ok {
+ pe.length += dpe.adjustLength(pe.length)
+ }
+
+ return nil
+}
+
+// we do not record metrics during the prep encoder pass
+func (pe *prepEncoder) metricRegistry() metrics.Registry {
+ return nil
+}
+
+func (pe *prepFlexibleEncoder) putArrayLength(in int) error {
+ pe.putUVarint(uint64(in + 1))
+ return nil
+}
+
+func (pe *prepFlexibleEncoder) putBytes(in []byte) error {
+ pe.putUVarint(uint64(len(in) + 1))
+ return pe.putRawBytes(in)
+}
+
+func (pe *prepFlexibleEncoder) putString(in string) error {
+ if err := pe.putArrayLength(len(in)); err != nil {
+ return err
+ }
+ return pe.putRawBytes([]byte(in))
+}
+
+func (pe *prepFlexibleEncoder) putNullableString(in *string) error {
+ if in == nil {
+ pe.putUVarint(0)
+ return nil
+ } else {
+ return pe.putString(*in)
+ }
+}
+
+func (pe *prepFlexibleEncoder) putStringArray(in []string) error {
+ err := pe.putArrayLength(len(in))
+ if err != nil {
+ return err
+ }
+
+ for _, str := range in {
+ if err := pe.putString(str); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (pe *prepFlexibleEncoder) putInt32Array(in []int32) error {
+ if in == nil {
+ return errors.New("expected int32 array to be non null")
+ }
+
+ pe.putUVarint(uint64(len(in)) + 1)
+ pe.length += 4 * len(in)
+ return nil
+}
+
+func (pe *prepFlexibleEncoder) putNullableInt32Array(in []int32) error {
+ if in == nil {
+ pe.putUVarint(0)
+ return nil
+ }
+
+ pe.putUVarint(uint64(len(in)) + 1)
+ pe.length += 4 * len(in)
+ return nil
+}
+
+func (pe *prepFlexibleEncoder) putEmptyTaggedFieldArray() {
+ pe.putUVarint(0)
+}
diff --git a/vendor/github.com/IBM/sarama/produce_request.go b/vendor/github.com/IBM/sarama/produce_request.go
new file mode 100644
index 0000000..de88ba4
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/produce_request.go
@@ -0,0 +1,274 @@
+package sarama
+
+import "github.com/rcrowley/go-metrics"
+
+// RequiredAcks is used in Produce Requests to tell the broker how many replica acknowledgements
+// it must see before responding. Any of the constants defined here are valid. On broker versions
+// prior to 0.8.2.0 any other positive int16 is also valid (the broker will wait for that many
+// acknowledgements) but in 0.8.2.0 and later this will raise an exception (it has been replaced
+// by setting the `min.isr` value in the brokers configuration).
+type RequiredAcks int16
+
+const (
+ // NoResponse doesn't send any response, the TCP ACK is all you get.
+ NoResponse RequiredAcks = 0
+ // WaitForLocal waits for only the local commit to succeed before responding.
+ WaitForLocal RequiredAcks = 1
+ // WaitForAll waits for all in-sync replicas to commit before responding.
+ // The minimum number of in-sync replicas is configured on the broker via
+ // the `min.insync.replicas` configuration key.
+ WaitForAll RequiredAcks = -1
+)
+
+type ProduceRequest struct {
+ TransactionalID *string
+ RequiredAcks RequiredAcks
+ Timeout int32
+ Version int16 // v1 requires Kafka 0.9, v2 requires Kafka 0.10, v3 requires Kafka 0.11
+ records map[string]map[int32]Records
+}
+
+func (r *ProduceRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func updateMsgSetMetrics(msgSet *MessageSet, compressionRatioMetric metrics.Histogram,
+ topicCompressionRatioMetric metrics.Histogram,
+) int64 {
+ var topicRecordCount int64
+ for _, messageBlock := range msgSet.Messages {
+ // Is this a fake "message" wrapping real messages?
+ if messageBlock.Msg.Set != nil {
+ topicRecordCount += int64(len(messageBlock.Msg.Set.Messages))
+ } else {
+ // A single uncompressed message
+ topicRecordCount++
+ }
+ // Better be safe than sorry when computing the compression ratio
+ if messageBlock.Msg.compressedSize != 0 {
+ compressionRatio := float64(len(messageBlock.Msg.Value)) /
+ float64(messageBlock.Msg.compressedSize)
+ // Histogram do not support decimal values, let's multiple it by 100 for better precision
+ intCompressionRatio := int64(100 * compressionRatio)
+ compressionRatioMetric.Update(intCompressionRatio)
+ topicCompressionRatioMetric.Update(intCompressionRatio)
+ }
+ }
+ return topicRecordCount
+}
+
+func updateBatchMetrics(recordBatch *RecordBatch, compressionRatioMetric metrics.Histogram,
+ topicCompressionRatioMetric metrics.Histogram,
+) int64 {
+ if recordBatch.compressedRecords != nil {
+ compressionRatio := int64(float64(recordBatch.recordsLen) / float64(len(recordBatch.compressedRecords)) * 100)
+ compressionRatioMetric.Update(compressionRatio)
+ topicCompressionRatioMetric.Update(compressionRatio)
+ }
+
+ return int64(len(recordBatch.Records))
+}
+
+func (r *ProduceRequest) encode(pe packetEncoder) error {
+ if r.Version >= 3 {
+ if err := pe.putNullableString(r.TransactionalID); err != nil {
+ return err
+ }
+ }
+ pe.putInt16(int16(r.RequiredAcks))
+ pe.putInt32(r.Timeout)
+ metricRegistry := pe.metricRegistry()
+ var batchSizeMetric metrics.Histogram
+ var compressionRatioMetric metrics.Histogram
+ if metricRegistry != nil {
+ batchSizeMetric = getOrRegisterHistogram("batch-size", metricRegistry)
+ compressionRatioMetric = getOrRegisterHistogram("compression-ratio", metricRegistry)
+ }
+ totalRecordCount := int64(0)
+
+ err := pe.putArrayLength(len(r.records))
+ if err != nil {
+ return err
+ }
+
+ for topic, partitions := range r.records {
+ err = pe.putString(topic)
+ if err != nil {
+ return err
+ }
+ err = pe.putArrayLength(len(partitions))
+ if err != nil {
+ return err
+ }
+ topicRecordCount := int64(0)
+ var topicCompressionRatioMetric metrics.Histogram
+ if metricRegistry != nil {
+ topicCompressionRatioMetric = getOrRegisterTopicHistogram("compression-ratio", topic, metricRegistry)
+ }
+ for id, records := range partitions {
+ startOffset := pe.offset()
+ pe.putInt32(id)
+ pe.push(&lengthField{})
+ err = records.encode(pe)
+ if err != nil {
+ return err
+ }
+ err = pe.pop()
+ if err != nil {
+ return err
+ }
+ if metricRegistry != nil {
+ if r.Version >= 3 {
+ topicRecordCount += updateBatchMetrics(records.RecordBatch, compressionRatioMetric, topicCompressionRatioMetric)
+ } else {
+ topicRecordCount += updateMsgSetMetrics(records.MsgSet, compressionRatioMetric, topicCompressionRatioMetric)
+ }
+ batchSize := int64(pe.offset() - startOffset)
+ batchSizeMetric.Update(batchSize)
+ getOrRegisterTopicHistogram("batch-size", topic, metricRegistry).Update(batchSize)
+ }
+ }
+ if topicRecordCount > 0 {
+ getOrRegisterTopicMeter("record-send-rate", topic, metricRegistry).Mark(topicRecordCount)
+ getOrRegisterTopicHistogram("records-per-request", topic, metricRegistry).Update(topicRecordCount)
+ totalRecordCount += topicRecordCount
+ }
+ }
+ if totalRecordCount > 0 {
+ metrics.GetOrRegisterMeter("record-send-rate", metricRegistry).Mark(totalRecordCount)
+ getOrRegisterHistogram("records-per-request", metricRegistry).Update(totalRecordCount)
+ }
+
+ return nil
+}
+
+func (r *ProduceRequest) decode(pd packetDecoder, version int16) error {
+ r.Version = version
+
+ if version >= 3 {
+ id, err := pd.getNullableString()
+ if err != nil {
+ return err
+ }
+ r.TransactionalID = id
+ }
+ requiredAcks, err := pd.getInt16()
+ if err != nil {
+ return err
+ }
+ r.RequiredAcks = RequiredAcks(requiredAcks)
+ if r.Timeout, err = pd.getInt32(); err != nil {
+ return err
+ }
+ topicCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if topicCount == 0 {
+ return nil
+ }
+
+ r.records = make(map[string]map[int32]Records)
+ for i := 0; i < topicCount; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+ partitionCount, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ r.records[topic] = make(map[int32]Records)
+
+ for j := 0; j < partitionCount; j++ {
+ partition, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ size, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ recordsDecoder, err := pd.getSubset(int(size))
+ if err != nil {
+ return err
+ }
+ var records Records
+ if err := records.decode(recordsDecoder); err != nil {
+ return err
+ }
+ r.records[topic][partition] = records
+ }
+ }
+
+ return nil
+}
+
+func (r *ProduceRequest) key() int16 {
+ return apiKeyProduce
+}
+
+func (r *ProduceRequest) version() int16 {
+ return r.Version
+}
+
+func (r *ProduceRequest) headerVersion() int16 {
+ return 1
+}
+
+func (r *ProduceRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 7
+}
+
+func (r *ProduceRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 7:
+ return V2_1_0_0
+ case 6:
+ return V2_0_0_0
+ case 4, 5:
+ return V1_0_0_0
+ case 3:
+ return V0_11_0_0
+ case 2:
+ return V0_10_0_0
+ case 1:
+ return V0_9_0_0
+ case 0:
+ return V0_8_2_0
+ default:
+ return V2_1_0_0
+ }
+}
+
+func (r *ProduceRequest) ensureRecords(topic string, partition int32) {
+ if r.records == nil {
+ r.records = make(map[string]map[int32]Records)
+ }
+
+ if r.records[topic] == nil {
+ r.records[topic] = make(map[int32]Records)
+ }
+}
+
+func (r *ProduceRequest) AddMessage(topic string, partition int32, msg *Message) {
+ r.ensureRecords(topic, partition)
+ set := r.records[topic][partition].MsgSet
+
+ if set == nil {
+ set = new(MessageSet)
+ r.records[topic][partition] = newLegacyRecords(set)
+ }
+
+ set.addMessage(msg)
+}
+
+func (r *ProduceRequest) AddSet(topic string, partition int32, set *MessageSet) {
+ r.ensureRecords(topic, partition)
+ r.records[topic][partition] = newLegacyRecords(set)
+}
+
+func (r *ProduceRequest) AddBatch(topic string, partition int32, batch *RecordBatch) {
+ r.ensureRecords(topic, partition)
+ r.records[topic][partition] = newDefaultRecords(batch)
+}
diff --git a/vendor/github.com/IBM/sarama/produce_response.go b/vendor/github.com/IBM/sarama/produce_response.go
new file mode 100644
index 0000000..f3c5d5d
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/produce_response.go
@@ -0,0 +1,237 @@
+package sarama
+
+import (
+ "fmt"
+ "time"
+)
+
+// Protocol, http://kafka.apache.org/protocol.html
+// v1
+// v2 = v3 = v4
+// v5 = v6 = v7
+// Produce Response (Version: 7) => [responses] throttle_time_ms
+// responses => topic [partition_responses]
+// topic => STRING
+// partition_responses => partition error_code base_offset log_append_time log_start_offset
+// partition => INT32
+// error_code => INT16
+// base_offset => INT64
+// log_append_time => INT64
+// log_start_offset => INT64
+// throttle_time_ms => INT32
+
+// partition_responses in protocol
+type ProduceResponseBlock struct {
+ Err KError // v0, error_code
+ Offset int64 // v0, base_offset
+ Timestamp time.Time // v2, log_append_time, and the broker is configured with `LogAppendTime`
+ StartOffset int64 // v5, log_start_offset
+}
+
+func (b *ProduceResponseBlock) decode(pd packetDecoder, version int16) (err error) {
+ b.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ b.Offset, err = pd.getInt64()
+ if err != nil {
+ return err
+ }
+
+ if version >= 2 {
+ if millis, err := pd.getInt64(); err != nil {
+ return err
+ } else if millis != -1 {
+ b.Timestamp = time.Unix(millis/1000, (millis%1000)*int64(time.Millisecond))
+ }
+ }
+
+ if version >= 5 {
+ b.StartOffset, err = pd.getInt64()
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (b *ProduceResponseBlock) encode(pe packetEncoder, version int16) (err error) {
+ pe.putKError(b.Err)
+ pe.putInt64(b.Offset)
+
+ if version >= 2 {
+ timestamp := int64(-1)
+ if !b.Timestamp.Before(time.Unix(0, 0)) {
+ timestamp = b.Timestamp.UnixNano() / int64(time.Millisecond)
+ } else if !b.Timestamp.IsZero() {
+ return PacketEncodingError{fmt.Sprintf("invalid timestamp (%v)", b.Timestamp)}
+ }
+ pe.putInt64(timestamp)
+ }
+
+ if version >= 5 {
+ pe.putInt64(b.StartOffset)
+ }
+
+ return nil
+}
+
+type ProduceResponse struct {
+ Blocks map[string]map[int32]*ProduceResponseBlock // v0, responses
+ Version int16
+ ThrottleTime time.Duration // v1, throttle_time_ms
+}
+
+func (r *ProduceResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *ProduceResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+
+ numTopics, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Blocks = make(map[string]map[int32]*ProduceResponseBlock, numTopics)
+ for i := 0; i < numTopics; i++ {
+ name, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ numBlocks, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ r.Blocks[name] = make(map[int32]*ProduceResponseBlock, numBlocks)
+
+ for j := 0; j < numBlocks; j++ {
+ id, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ block := new(ProduceResponseBlock)
+ err = block.decode(pd, version)
+ if err != nil {
+ return err
+ }
+ r.Blocks[name][id] = block
+ }
+ }
+
+ if r.Version >= 1 {
+ if r.ThrottleTime, err = pd.getDurationMs(); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (r *ProduceResponse) encode(pe packetEncoder) error {
+ err := pe.putArrayLength(len(r.Blocks))
+ if err != nil {
+ return err
+ }
+ for topic, partitions := range r.Blocks {
+ err = pe.putString(topic)
+ if err != nil {
+ return err
+ }
+ err = pe.putArrayLength(len(partitions))
+ if err != nil {
+ return err
+ }
+ for id, prb := range partitions {
+ pe.putInt32(id)
+ err = prb.encode(pe, r.Version)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ if r.Version >= 1 {
+ pe.putDurationMs(r.ThrottleTime)
+ }
+ return nil
+}
+
+func (r *ProduceResponse) key() int16 {
+ return apiKeyProduce
+}
+
+func (r *ProduceResponse) version() int16 {
+ return r.Version
+}
+
+func (r *ProduceResponse) headerVersion() int16 {
+ return 0
+}
+
+func (r *ProduceResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 7
+}
+
+func (r *ProduceResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 7:
+ return V2_1_0_0
+ case 6:
+ return V2_0_0_0
+ case 4, 5:
+ return V1_0_0_0
+ case 3:
+ return V0_11_0_0
+ case 2:
+ return V0_10_0_0
+ case 1:
+ return V0_9_0_0
+ case 0:
+ return V0_8_2_0
+ default:
+ return V2_1_0_0
+ }
+}
+
+func (r *ProduceResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
+
+func (r *ProduceResponse) GetBlock(topic string, partition int32) *ProduceResponseBlock {
+ if r.Blocks == nil {
+ return nil
+ }
+
+ if r.Blocks[topic] == nil {
+ return nil
+ }
+
+ return r.Blocks[topic][partition]
+}
+
+// Testing API
+
+func (r *ProduceResponse) AddTopicPartition(topic string, partition int32, err KError) {
+ if r.Blocks == nil {
+ r.Blocks = make(map[string]map[int32]*ProduceResponseBlock)
+ }
+ byTopic, ok := r.Blocks[topic]
+ if !ok {
+ byTopic = make(map[int32]*ProduceResponseBlock)
+ r.Blocks[topic] = byTopic
+ }
+ block := &ProduceResponseBlock{
+ Err: err,
+ }
+ if r.Version >= 2 {
+ block.Timestamp = time.Now()
+ }
+ byTopic[partition] = block
+}
diff --git a/vendor/github.com/IBM/sarama/produce_set.go b/vendor/github.com/IBM/sarama/produce_set.go
new file mode 100644
index 0000000..c91403d
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/produce_set.go
@@ -0,0 +1,289 @@
+package sarama
+
+import (
+ "encoding/binary"
+ "errors"
+ "time"
+)
+
+type partitionSet struct {
+ msgs []*ProducerMessage
+ recordsToSend Records
+ bufferBytes int
+}
+
+type produceSet struct {
+ parent *asyncProducer
+ msgs map[string]map[int32]*partitionSet
+ producerID int64
+ producerEpoch int16
+
+ bufferBytes int
+ bufferCount int
+}
+
+func newProduceSet(parent *asyncProducer) *produceSet {
+ pid, epoch := parent.txnmgr.getProducerID()
+ return &produceSet{
+ msgs: make(map[string]map[int32]*partitionSet),
+ parent: parent,
+ producerID: pid,
+ producerEpoch: epoch,
+ }
+}
+
+func (ps *produceSet) add(msg *ProducerMessage) error {
+ var err error
+ var key, val []byte
+
+ if msg.Key != nil {
+ if key, err = msg.Key.Encode(); err != nil {
+ return err
+ }
+ }
+
+ if msg.Value != nil {
+ if val, err = msg.Value.Encode(); err != nil {
+ return err
+ }
+ }
+
+ timestamp := msg.Timestamp
+ if timestamp.IsZero() {
+ timestamp = time.Now()
+ }
+ timestamp = timestamp.Truncate(time.Millisecond)
+
+ partitions := ps.msgs[msg.Topic]
+ if partitions == nil {
+ partitions = make(map[int32]*partitionSet)
+ ps.msgs[msg.Topic] = partitions
+ }
+
+ var size int
+
+ set := partitions[msg.Partition]
+ if set == nil {
+ if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) {
+ batch := &RecordBatch{
+ FirstTimestamp: timestamp,
+ Version: 2,
+ Codec: ps.parent.conf.Producer.Compression,
+ CompressionLevel: ps.parent.conf.Producer.CompressionLevel,
+ ProducerID: ps.producerID,
+ ProducerEpoch: ps.producerEpoch,
+ }
+ if ps.parent.conf.Producer.Idempotent {
+ batch.FirstSequence = msg.sequenceNumber
+ }
+ set = &partitionSet{recordsToSend: newDefaultRecords(batch)}
+ size = recordBatchOverhead
+ } else {
+ set = &partitionSet{recordsToSend: newLegacyRecords(new(MessageSet))}
+ }
+ partitions[msg.Partition] = set
+ }
+
+ if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) {
+ if ps.parent.conf.Producer.Idempotent && msg.sequenceNumber < set.recordsToSend.RecordBatch.FirstSequence {
+ return errors.New("assertion failed: message out of sequence added to a batch")
+ }
+ }
+
+ // Past this point we can't return an error, because we've already added the message to the set.
+ set.msgs = append(set.msgs, msg)
+
+ if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) {
+ // We are being conservative here to avoid having to prep encode the record
+ size += maximumRecordOverhead
+ rec := &Record{
+ Key: key,
+ Value: val,
+ TimestampDelta: timestamp.Sub(set.recordsToSend.RecordBatch.FirstTimestamp),
+ }
+ size += len(key) + len(val)
+ if len(msg.Headers) > 0 {
+ rec.Headers = make([]*RecordHeader, len(msg.Headers))
+ for i := range msg.Headers {
+ rec.Headers[i] = &msg.Headers[i]
+ size += len(rec.Headers[i].Key) + len(rec.Headers[i].Value) + 2*binary.MaxVarintLen32
+ }
+ }
+ set.recordsToSend.RecordBatch.addRecord(rec)
+ } else {
+ msgToSend := &Message{Codec: CompressionNone, Key: key, Value: val}
+ if ps.parent.conf.Version.IsAtLeast(V0_10_0_0) {
+ msgToSend.Timestamp = timestamp
+ msgToSend.Version = 1
+ }
+ set.recordsToSend.MsgSet.addMessage(msgToSend)
+ size = producerMessageOverhead + len(key) + len(val)
+ }
+
+ set.bufferBytes += size
+ ps.bufferBytes += size
+ ps.bufferCount++
+
+ return nil
+}
+
+func (ps *produceSet) buildRequest() *ProduceRequest {
+ req := &ProduceRequest{
+ RequiredAcks: ps.parent.conf.Producer.RequiredAcks,
+ Timeout: int32(ps.parent.conf.Producer.Timeout / time.Millisecond),
+ }
+ if ps.parent.conf.Version.IsAtLeast(V0_10_0_0) {
+ req.Version = 2
+ }
+ if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) {
+ req.Version = 3
+ if ps.parent.IsTransactional() {
+ req.TransactionalID = &ps.parent.conf.Producer.Transaction.ID
+ }
+ }
+ if ps.parent.conf.Version.IsAtLeast(V1_0_0_0) {
+ req.Version = 5
+ }
+ if ps.parent.conf.Version.IsAtLeast(V2_0_0_0) {
+ req.Version = 6
+ }
+ if ps.parent.conf.Version.IsAtLeast(V2_1_0_0) {
+ req.Version = 7
+ }
+
+ for topic, partitionSets := range ps.msgs {
+ for partition, set := range partitionSets {
+ if req.Version >= 3 {
+ // If the API version we're hitting is 3 or greater, we need to calculate
+ // offsets for each record in the batch relative to FirstOffset.
+ // Additionally, we must set LastOffsetDelta to the value of the last offset
+ // in the batch. Since the OffsetDelta of the first record is 0, we know that the
+ // final record of any batch will have an offset of (# of records in batch) - 1.
+ // (See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets
+ // under the RecordBatch section for details.)
+ rb := set.recordsToSend.RecordBatch
+ if len(rb.Records) > 0 {
+ rb.LastOffsetDelta = int32(len(rb.Records) - 1)
+ var maxTimestampDelta time.Duration
+ for i, record := range rb.Records {
+ record.OffsetDelta = int64(i)
+ maxTimestampDelta = max(maxTimestampDelta, record.TimestampDelta)
+ }
+ // Also set the MaxTimestamp similar to other clients.
+ rb.MaxTimestamp = rb.FirstTimestamp.Add(maxTimestampDelta)
+ }
+
+ // Set the batch as transactional when a transactionalID is set
+ rb.IsTransactional = ps.parent.IsTransactional()
+
+ req.AddBatch(topic, partition, rb)
+ continue
+ }
+ if ps.parent.conf.Producer.Compression == CompressionNone {
+ req.AddSet(topic, partition, set.recordsToSend.MsgSet)
+ } else {
+ // When compression is enabled, the entire set for each partition is compressed
+ // and sent as the payload of a single fake "message" with the appropriate codec
+ // set and no key. When the server sees a message with a compression codec, it
+ // decompresses the payload and treats the result as its message set.
+
+ if ps.parent.conf.Version.IsAtLeast(V0_10_0_0) {
+ // If our version is 0.10 or later, assign relative offsets
+ // to the inner messages. This lets the broker avoid
+ // recompressing the message set.
+ // (See https://cwiki.apache.org/confluence/display/KAFKA/KIP-31+-+Move+to+relative+offsets+in+compressed+message+sets
+ // for details on relative offsets.)
+ for i, msg := range set.recordsToSend.MsgSet.Messages {
+ msg.Offset = int64(i)
+ }
+ }
+ payload, err := encode(set.recordsToSend.MsgSet, ps.parent.metricsRegistry)
+ if err != nil {
+ Logger.Println(err) // if this happens, it's basically our fault.
+ panic(err)
+ }
+ compMsg := &Message{
+ Codec: ps.parent.conf.Producer.Compression,
+ CompressionLevel: ps.parent.conf.Producer.CompressionLevel,
+ Key: nil,
+ Value: payload,
+ Set: set.recordsToSend.MsgSet, // Provide the underlying message set for accurate metrics
+ }
+ if ps.parent.conf.Version.IsAtLeast(V0_10_0_0) {
+ compMsg.Version = 1
+ compMsg.Timestamp = set.recordsToSend.MsgSet.Messages[0].Msg.Timestamp
+ }
+ req.AddMessage(topic, partition, compMsg)
+ }
+ }
+ }
+
+ return req
+}
+
+func (ps *produceSet) eachPartition(cb func(topic string, partition int32, pSet *partitionSet)) {
+ for topic, partitionSet := range ps.msgs {
+ for partition, set := range partitionSet {
+ cb(topic, partition, set)
+ }
+ }
+}
+
+func (ps *produceSet) dropPartition(topic string, partition int32) []*ProducerMessage {
+ if ps.msgs[topic] == nil {
+ return nil
+ }
+ set := ps.msgs[topic][partition]
+ if set == nil {
+ return nil
+ }
+ ps.bufferBytes -= set.bufferBytes
+ ps.bufferCount -= len(set.msgs)
+ delete(ps.msgs[topic], partition)
+ return set.msgs
+}
+
+func (ps *produceSet) wouldOverflow(msg *ProducerMessage) bool {
+ version := 1
+ if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) {
+ version = 2
+ }
+
+ switch {
+ // Would we overflow our maximum possible size-on-the-wire? 10KiB is arbitrary overhead for safety.
+ case ps.bufferBytes+msg.ByteSize(version) >= int(MaxRequestSize-(10*1024)):
+ return true
+ // Would we overflow the size-limit of a message-batch for this partition?
+ case ps.msgs[msg.Topic] != nil && ps.msgs[msg.Topic][msg.Partition] != nil &&
+ ps.msgs[msg.Topic][msg.Partition].bufferBytes+msg.ByteSize(version) >= ps.parent.conf.Producer.MaxMessageBytes:
+ return true
+ // Would we overflow simply in number of messages?
+ case ps.parent.conf.Producer.Flush.MaxMessages > 0 && ps.bufferCount >= ps.parent.conf.Producer.Flush.MaxMessages:
+ return true
+ default:
+ return false
+ }
+}
+
+func (ps *produceSet) readyToFlush() bool {
+ switch {
+ // If we don't have any messages, nothing else matters
+ case ps.empty():
+ return false
+ // If all three config values are 0, we always flush as-fast-as-possible
+ case ps.parent.conf.Producer.Flush.Frequency == 0 && ps.parent.conf.Producer.Flush.Bytes == 0 && ps.parent.conf.Producer.Flush.Messages == 0:
+ return true
+ // If we've passed the message trigger-point
+ case ps.parent.conf.Producer.Flush.Messages > 0 && ps.bufferCount >= ps.parent.conf.Producer.Flush.Messages:
+ return true
+ // If we've passed the byte trigger-point
+ case ps.parent.conf.Producer.Flush.Bytes > 0 && ps.bufferBytes >= ps.parent.conf.Producer.Flush.Bytes:
+ return true
+ default:
+ return false
+ }
+}
+
+func (ps *produceSet) empty() bool {
+ return ps.bufferCount == 0
+}
diff --git a/vendor/github.com/IBM/sarama/quota_types.go b/vendor/github.com/IBM/sarama/quota_types.go
new file mode 100644
index 0000000..4f33af0
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/quota_types.go
@@ -0,0 +1,21 @@
+package sarama
+
+type (
+ QuotaEntityType string
+
+ QuotaMatchType int
+)
+
+// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaEntity.java
+const (
+ QuotaEntityUser QuotaEntityType = "user"
+ QuotaEntityClientID QuotaEntityType = "client-id"
+ QuotaEntityIP QuotaEntityType = "ip"
+)
+
+// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java
+const (
+ QuotaMatchExact QuotaMatchType = iota
+ QuotaMatchDefault
+ QuotaMatchAny
+)
diff --git a/vendor/github.com/IBM/sarama/real_decoder.go b/vendor/github.com/IBM/sarama/real_decoder.go
new file mode 100644
index 0000000..ddeb543
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/real_decoder.go
@@ -0,0 +1,549 @@
+package sarama
+
+import (
+ "encoding/binary"
+ "math"
+ "time"
+
+ "github.com/rcrowley/go-metrics"
+)
+
+var (
+ errInvalidArrayLength = PacketDecodingError{"invalid array length"}
+ errInvalidByteSliceLength = PacketDecodingError{"invalid byteslice length"}
+ errInvalidStringLength = PacketDecodingError{"invalid string length"}
+ errVarintOverflow = PacketDecodingError{"varint overflow"}
+ errUVarintOverflow = PacketDecodingError{"uvarint overflow"}
+ errInvalidBool = PacketDecodingError{"invalid bool"}
+)
+
+type realDecoder struct {
+ raw []byte
+ off int
+ stack []pushDecoder
+ registry metrics.Registry
+}
+
+type realFlexibleDecoder struct {
+ *realDecoder
+}
+
+// primitives
+
+func (rd *realDecoder) getInt8() (int8, error) {
+ if rd.remaining() < 1 {
+ rd.off = len(rd.raw)
+ return -1, ErrInsufficientData
+ }
+ tmp := int8(rd.raw[rd.off])
+ rd.off++
+ return tmp, nil
+}
+
+func (rd *realDecoder) getInt16() (int16, error) {
+ if rd.remaining() < 2 {
+ rd.off = len(rd.raw)
+ return -1, ErrInsufficientData
+ }
+ tmp := int16(binary.BigEndian.Uint16(rd.raw[rd.off:]))
+ rd.off += 2
+ return tmp, nil
+}
+
+func (rd *realDecoder) getInt32() (int32, error) {
+ if rd.remaining() < 4 {
+ rd.off = len(rd.raw)
+ return -1, ErrInsufficientData
+ }
+ tmp := int32(binary.BigEndian.Uint32(rd.raw[rd.off:]))
+ rd.off += 4
+ return tmp, nil
+}
+
+func (rd *realDecoder) getInt64() (int64, error) {
+ if rd.remaining() < 8 {
+ rd.off = len(rd.raw)
+ return -1, ErrInsufficientData
+ }
+ tmp := int64(binary.BigEndian.Uint64(rd.raw[rd.off:]))
+ rd.off += 8
+ return tmp, nil
+}
+
+func (rd *realDecoder) getVarint() (int64, error) {
+ tmp, n := binary.Varint(rd.raw[rd.off:])
+ if n == 0 {
+ rd.off = len(rd.raw)
+ return -1, ErrInsufficientData
+ }
+ if n < 0 {
+ rd.off -= n
+ return -1, errVarintOverflow
+ }
+ rd.off += n
+ return tmp, nil
+}
+
+func (rd *realDecoder) getUVarint() (uint64, error) {
+ tmp, n := binary.Uvarint(rd.raw[rd.off:])
+ if n == 0 {
+ rd.off = len(rd.raw)
+ return 0, ErrInsufficientData
+ }
+
+ if n < 0 {
+ rd.off -= n
+ return 0, errUVarintOverflow
+ }
+
+ rd.off += n
+ return tmp, nil
+}
+
+func (rd *realDecoder) getFloat64() (float64, error) {
+ if rd.remaining() < 8 {
+ rd.off = len(rd.raw)
+ return -1, ErrInsufficientData
+ }
+ tmp := math.Float64frombits(binary.BigEndian.Uint64(rd.raw[rd.off:]))
+ rd.off += 8
+ return tmp, nil
+}
+
+func (rd *realDecoder) getArrayLength() (int, error) {
+ if rd.remaining() < 4 {
+ rd.off = len(rd.raw)
+ return -1, ErrInsufficientData
+ }
+ // cast to int32 first to get correct signedness for length before then
+ // casting to int for ease of interop
+ tmp := int(int32(binary.BigEndian.Uint32(rd.raw[rd.off:])))
+ rd.off += 4
+ if tmp > rd.remaining() {
+ rd.off = len(rd.raw)
+ return -1, ErrInsufficientData
+ } else if tmp > int(MaxResponseSize) {
+ return -1, errInvalidArrayLength
+ }
+ return tmp, nil
+}
+
+func (rd *realDecoder) getBool() (bool, error) {
+ b, err := rd.getInt8()
+ if err != nil || b == 0 {
+ return false, err
+ }
+ if b != 1 {
+ return false, errInvalidBool
+ }
+ return true, nil
+}
+
+func (rd *realDecoder) getKError() (KError, error) {
+ i, err := rd.getInt16()
+ return KError(i), err
+}
+
+func (rd *realDecoder) getDurationMs() (time.Duration, error) {
+ t, err := rd.getInt32()
+ if err != nil {
+ return time.Duration(0), err
+ }
+ return time.Duration(t) * time.Millisecond, nil
+}
+
+func (rd *realDecoder) getTaggedFieldArray(decoders taggedFieldDecoders) error {
+ return PacketDecodingError{"tagged fields used in non-flexible context"}
+}
+
+func (rd *realDecoder) getEmptyTaggedFieldArray() (int, error) {
+ return 0, nil
+}
+
+// collections
+
+func (rd *realDecoder) getBytes() ([]byte, error) {
+ tmp, err := rd.getInt32()
+ if err != nil {
+ return nil, err
+ }
+ if tmp == -1 {
+ return nil, nil
+ }
+
+ return rd.getRawBytes(int(tmp))
+}
+
+func (rd *realDecoder) getVarintBytes() ([]byte, error) {
+ tmp, err := rd.getVarint()
+ if err != nil {
+ return nil, err
+ }
+ if tmp == -1 {
+ return nil, nil
+ }
+
+ return rd.getRawBytes(int(tmp))
+}
+
+func (rd *realDecoder) getStringLength() (int, error) {
+ length, err := rd.getInt16()
+ if err != nil {
+ return 0, err
+ }
+
+ n := int(length)
+
+ switch {
+ case n < -1:
+ return 0, errInvalidStringLength
+ case n > rd.remaining():
+ rd.off = len(rd.raw)
+ return 0, ErrInsufficientData
+ }
+
+ return n, nil
+}
+
+func (rd *realDecoder) getString() (string, error) {
+ n, err := rd.getStringLength()
+ if err != nil || n == -1 {
+ return "", err
+ }
+
+ tmpStr := string(rd.raw[rd.off : rd.off+n])
+ rd.off += n
+ return tmpStr, nil
+}
+
+func (rd *realDecoder) getNullableString() (*string, error) {
+ n, err := rd.getStringLength()
+ if err != nil || n == -1 {
+ return nil, err
+ }
+
+ tmpStr := string(rd.raw[rd.off : rd.off+n])
+ rd.off += n
+ return &tmpStr, err
+}
+
+func (rd *realDecoder) getInt32Array() ([]int32, error) {
+ n, err := rd.getArrayLength()
+ if err != nil {
+ return nil, err
+ }
+ if n <= 0 {
+ return nil, nil
+ }
+
+ if rd.remaining() < 4*n {
+ rd.off = len(rd.raw)
+ return nil, ErrInsufficientData
+ }
+
+ ret := make([]int32, n)
+ for i := range ret {
+ ret[i] = int32(binary.BigEndian.Uint32(rd.raw[rd.off:]))
+ rd.off += 4
+ }
+ return ret, nil
+}
+
+func (rd *realDecoder) getInt64Array() ([]int64, error) {
+ n, err := rd.getArrayLength()
+ if err != nil {
+ return nil, err
+ }
+ if n <= 0 {
+ return nil, nil
+ }
+
+ if rd.remaining() < 8*n {
+ rd.off = len(rd.raw)
+ return nil, ErrInsufficientData
+ }
+
+ ret := make([]int64, n)
+ for i := range ret {
+ ret[i] = int64(binary.BigEndian.Uint64(rd.raw[rd.off:]))
+ rd.off += 8
+ }
+ return ret, nil
+}
+
+func (rd *realDecoder) getStringArray() ([]string, error) {
+ n, err := rd.getArrayLength()
+ if err != nil {
+ return nil, err
+ }
+ if n <= 0 {
+ return nil, nil
+ }
+
+ ret := make([]string, n)
+ for i := range ret {
+ str, err := rd.getString()
+ if err != nil {
+ return nil, err
+ }
+
+ ret[i] = str
+ }
+ return ret, nil
+}
+
+// subsets
+
+func (rd *realDecoder) remaining() int {
+ return len(rd.raw) - rd.off
+}
+
+func (rd *realDecoder) getSubset(length int) (packetDecoder, error) {
+ buf, err := rd.getRawBytes(length)
+ if err != nil {
+ return nil, err
+ }
+ return &realDecoder{raw: buf}, nil
+}
+
+func (rd *realDecoder) getRawBytes(length int) ([]byte, error) {
+ if length < 0 {
+ return nil, errInvalidByteSliceLength
+ } else if length > rd.remaining() {
+ rd.off = len(rd.raw)
+ return nil, ErrInsufficientData
+ }
+
+ start := rd.off
+ rd.off += length
+ return rd.raw[start:rd.off], nil
+}
+
+func (rd *realDecoder) peek(offset, length int) (packetDecoder, error) {
+ if rd.remaining() < offset+length {
+ return nil, ErrInsufficientData
+ }
+ off := rd.off + offset
+ return &realDecoder{raw: rd.raw[off : off+length]}, nil
+}
+
+func (rd *realDecoder) peekInt8(offset int) (int8, error) {
+ const byteLen = 1
+ if rd.remaining() < offset+byteLen {
+ return -1, ErrInsufficientData
+ }
+ return int8(rd.raw[rd.off+offset]), nil
+}
+
+// stacks
+
+func (rd *realDecoder) push(in pushDecoder) error {
+ in.saveOffset(rd.off)
+
+ var reserve int
+ if dpd, ok := in.(dynamicPushDecoder); ok {
+ if err := dpd.decode(rd); err != nil {
+ return err
+ }
+ } else {
+ reserve = in.reserveLength()
+ if rd.remaining() < reserve {
+ rd.off = len(rd.raw)
+ return ErrInsufficientData
+ }
+ }
+
+ rd.stack = append(rd.stack, in)
+
+ rd.off += reserve
+
+ return nil
+}
+
+func (rd *realDecoder) pop() error {
+ // this is go's ugly pop pattern (the inverse of append)
+ in := rd.stack[len(rd.stack)-1]
+ rd.stack = rd.stack[:len(rd.stack)-1]
+
+ return in.check(rd.off, rd.raw)
+}
+
+func (rd *realDecoder) metricRegistry() metrics.Registry {
+ return rd.registry
+}
+
+func (rd *realFlexibleDecoder) getArrayLength() (int, error) {
+ n, err := rd.getUVarint()
+ if err != nil {
+ return 0, err
+ }
+
+ if n == 0 {
+ return 0, nil
+ }
+
+ return int(n) - 1, nil
+}
+
+func (rd *realFlexibleDecoder) getEmptyTaggedFieldArray() (int, error) {
+ tagCount, err := rd.getUVarint()
+ if err != nil {
+ return 0, err
+ }
+
+ // skip over any tagged fields without deserializing them
+ // as we don't currently support doing anything with them
+ for i := uint64(0); i < tagCount; i++ {
+ // fetch and ignore tag identifier
+ _, err := rd.getUVarint()
+ if err != nil {
+ return 0, err
+ }
+ length, err := rd.getUVarint()
+ if err != nil {
+ return 0, err
+ }
+ if _, err := rd.getRawBytes(int(length)); err != nil {
+ return 0, err
+ }
+ }
+
+ return 0, nil
+}
+
+func (rd *realFlexibleDecoder) getTaggedFieldArray(decoders taggedFieldDecoders) error {
+ // if we have no decoders just skip over the tagged fields
+ if decoders == nil {
+ _, err := rd.getEmptyTaggedFieldArray()
+ return err
+ }
+
+ tagCount, err := rd.getUVarint()
+ if err != nil {
+ return err
+ }
+
+ for i := uint64(0); i < tagCount; i++ {
+ // fetch and ignore tag identifier
+ id, err := rd.getUVarint()
+ if err != nil {
+ return err
+ }
+ length, err := rd.getUVarint()
+ if err != nil {
+ return err
+ }
+ bytes, err := rd.getRawBytes(int(length))
+ if err != nil {
+ return err
+ }
+ decoder, ok := decoders[id]
+ if !ok {
+ continue
+ }
+ if err := decoder(&realFlexibleDecoder{&realDecoder{raw: bytes}}); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (rd *realFlexibleDecoder) getBytes() ([]byte, error) {
+ n, err := rd.getUVarint()
+ if err != nil {
+ return nil, err
+ }
+
+ length := int(n - 1)
+ return rd.getRawBytes(length)
+}
+
+func (rd *realFlexibleDecoder) getStringLength() (int, error) {
+ length, err := rd.getUVarint()
+ if err != nil {
+ return 0, err
+ }
+
+ n := int(length - 1)
+
+ switch {
+ case n < -1:
+ return 0, errInvalidStringLength
+ case n > rd.remaining():
+ rd.off = len(rd.raw)
+ return 0, ErrInsufficientData
+ }
+
+ return n, nil
+}
+
+func (rd *realFlexibleDecoder) getString() (string, error) {
+ length, err := rd.getStringLength()
+ if err != nil || length == -1 {
+ return "", err
+ }
+
+ if length < 0 {
+ return "", errInvalidStringLength
+ }
+ tmpStr := string(rd.raw[rd.off : rd.off+length])
+ rd.off += length
+ return tmpStr, nil
+}
+
+func (rd *realFlexibleDecoder) getNullableString() (*string, error) {
+ length, err := rd.getStringLength()
+ if err != nil {
+ return nil, err
+ }
+
+ if length < 0 {
+ return nil, err
+ }
+
+ tmpStr := string(rd.raw[rd.off : rd.off+length])
+ rd.off += length
+ return &tmpStr, err
+}
+
+func (rd *realFlexibleDecoder) getInt32Array() ([]int32, error) {
+ n, err := rd.getUVarint()
+ if err != nil {
+ return nil, err
+ }
+
+ if n == 0 {
+ return nil, nil
+ }
+
+ arrayLength := int(n) - 1
+
+ ret := make([]int32, arrayLength)
+
+ for i := range ret {
+ ret[i] = int32(binary.BigEndian.Uint32(rd.raw[rd.off:]))
+ rd.off += 4
+ }
+ return ret, nil
+}
+
+func (rd *realFlexibleDecoder) getStringArray() ([]string, error) {
+ n, err := rd.getArrayLength()
+ if err != nil {
+ return nil, err
+ }
+ if n <= 0 {
+ return nil, nil
+ }
+
+ ret := make([]string, n)
+ for i := range ret {
+ str, err := rd.getString()
+ if err != nil {
+ return nil, err
+ }
+
+ ret[i] = str
+ }
+ return ret, nil
+}
diff --git a/vendor/github.com/IBM/sarama/real_encoder.go b/vendor/github.com/IBM/sarama/real_encoder.go
new file mode 100644
index 0000000..b9bd178
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/real_encoder.go
@@ -0,0 +1,268 @@
+package sarama
+
+import (
+ "encoding/binary"
+ "errors"
+ "math"
+ "time"
+
+ "github.com/rcrowley/go-metrics"
+)
+
+type realEncoder struct {
+ raw []byte
+ off int
+ stack []pushEncoder
+ registry metrics.Registry
+}
+
+type realFlexibleEncoder struct {
+ *realEncoder
+}
+
+// primitives
+
+func (re *realEncoder) putInt8(in int8) {
+ re.raw[re.off] = byte(in)
+ re.off++
+}
+
+func (re *realEncoder) putInt16(in int16) {
+ binary.BigEndian.PutUint16(re.raw[re.off:], uint16(in))
+ re.off += 2
+}
+
+func (re *realEncoder) putInt32(in int32) {
+ binary.BigEndian.PutUint32(re.raw[re.off:], uint32(in))
+ re.off += 4
+}
+
+func (re *realEncoder) putInt64(in int64) {
+ binary.BigEndian.PutUint64(re.raw[re.off:], uint64(in))
+ re.off += 8
+}
+
+func (re *realEncoder) putVarint(in int64) {
+ re.off += binary.PutVarint(re.raw[re.off:], in)
+}
+
+func (re *realEncoder) putUVarint(in uint64) {
+ re.off += binary.PutUvarint(re.raw[re.off:], in)
+}
+
+func (re *realEncoder) putFloat64(in float64) {
+ binary.BigEndian.PutUint64(re.raw[re.off:], math.Float64bits(in))
+ re.off += 8
+}
+
+func (re *realEncoder) putArrayLength(in int) error {
+ re.putInt32(int32(in))
+ return nil
+}
+
+func (re *realEncoder) putBool(in bool) {
+ if in {
+ re.putInt8(1)
+ return
+ }
+ re.putInt8(0)
+}
+
+func (re *realEncoder) putKError(in KError) {
+ re.putInt16(int16(in))
+}
+
+func (re *realEncoder) putDurationMs(in time.Duration) {
+ re.putInt32(int32(in / time.Millisecond))
+}
+
+// collection
+
+func (re *realEncoder) putRawBytes(in []byte) error {
+ copy(re.raw[re.off:], in)
+ re.off += len(in)
+ return nil
+}
+
+func (re *realEncoder) putBytes(in []byte) error {
+ if in == nil {
+ re.putInt32(-1)
+ return nil
+ }
+ re.putInt32(int32(len(in)))
+ return re.putRawBytes(in)
+}
+
+func (re *realEncoder) putVarintBytes(in []byte) error {
+ if in == nil {
+ re.putVarint(-1)
+ return nil
+ }
+ re.putVarint(int64(len(in)))
+ return re.putRawBytes(in)
+}
+
+func (re *realEncoder) putString(in string) error {
+ re.putInt16(int16(len(in)))
+ copy(re.raw[re.off:], in)
+ re.off += len(in)
+ return nil
+}
+
+func (re *realEncoder) putNullableString(in *string) error {
+ if in == nil {
+ re.putInt16(-1)
+ return nil
+ }
+ return re.putString(*in)
+}
+
+func (re *realEncoder) putStringArray(in []string) error {
+ err := re.putArrayLength(len(in))
+ if err != nil {
+ return err
+ }
+
+ for _, val := range in {
+ if err := re.putString(val); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (re *realEncoder) putInt32Array(in []int32) error {
+ err := re.putArrayLength(len(in))
+ if err != nil {
+ return err
+ }
+ for _, val := range in {
+ re.putInt32(val)
+ }
+ return nil
+}
+
+func (re *realEncoder) putNullableInt32Array(in []int32) error {
+ if in == nil {
+ re.putInt32(-1)
+ return nil
+ }
+ err := re.putArrayLength(len(in))
+ if err != nil {
+ return err
+ }
+ for _, val := range in {
+ re.putInt32(val)
+ }
+ return nil
+}
+
+func (re *realEncoder) putInt64Array(in []int64) error {
+ err := re.putArrayLength(len(in))
+ if err != nil {
+ return err
+ }
+ for _, val := range in {
+ re.putInt64(val)
+ }
+ return nil
+}
+
+func (re *realEncoder) putEmptyTaggedFieldArray() {
+}
+
+func (re *realEncoder) offset() int {
+ return re.off
+}
+
+// stacks
+
+func (re *realEncoder) push(in pushEncoder) {
+ in.saveOffset(re.off)
+ re.off += in.reserveLength()
+ re.stack = append(re.stack, in)
+}
+
+func (re *realEncoder) pop() error {
+ // this is go's ugly pop pattern (the inverse of append)
+ in := re.stack[len(re.stack)-1]
+ re.stack = re.stack[:len(re.stack)-1]
+
+ return in.run(re.off, re.raw)
+}
+
+// we do record metrics during the real encoder pass
+func (re *realEncoder) metricRegistry() metrics.Registry {
+ return re.registry
+}
+
+func (re *realFlexibleEncoder) putArrayLength(in int) error {
+ // 0 represents a null array, so +1 has to be added
+ re.putUVarint(uint64(in + 1))
+ return nil
+}
+
+func (re *realFlexibleEncoder) putBytes(in []byte) error {
+ re.putUVarint(uint64(len(in) + 1))
+ return re.putRawBytes(in)
+}
+
+func (re *realFlexibleEncoder) putString(in string) error {
+ if err := re.putArrayLength(len(in)); err != nil {
+ return err
+ }
+ return re.putRawBytes([]byte(in))
+}
+
+func (re *realFlexibleEncoder) putNullableString(in *string) error {
+ if in == nil {
+ re.putInt8(0)
+ return nil
+ }
+ return re.putString(*in)
+}
+
+func (re *realFlexibleEncoder) putStringArray(in []string) error {
+ err := re.putArrayLength(len(in))
+ if err != nil {
+ return err
+ }
+
+ for _, val := range in {
+ if err := re.putString(val); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (re *realFlexibleEncoder) putInt32Array(in []int32) error {
+ if in == nil {
+ return errors.New("expected int32 array to be non null")
+ }
+ // 0 represents a null array, so +1 has to be added
+ re.putUVarint(uint64(len(in)) + 1)
+ for _, val := range in {
+ re.putInt32(val)
+ }
+ return nil
+}
+
+func (re *realFlexibleEncoder) putNullableInt32Array(in []int32) error {
+ if in == nil {
+ re.putUVarint(0)
+ return nil
+ }
+ // 0 represents a null array, so +1 has to be added
+ re.putUVarint(uint64(len(in)) + 1)
+ for _, val := range in {
+ re.putInt32(val)
+ }
+ return nil
+}
+
+func (re *realFlexibleEncoder) putEmptyTaggedFieldArray() {
+ re.putUVarint(0)
+}
diff --git a/vendor/github.com/IBM/sarama/record.go b/vendor/github.com/IBM/sarama/record.go
new file mode 100644
index 0000000..a3fe8c0
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/record.go
@@ -0,0 +1,116 @@
+package sarama
+
+import (
+ "encoding/binary"
+ "time"
+)
+
+const (
+ isTransactionalMask = 0x10
+ controlMask = 0x20
+ maximumRecordOverhead = 5*binary.MaxVarintLen32 + binary.MaxVarintLen64 + 1
+)
+
+// RecordHeader stores key and value for a record header
+type RecordHeader struct {
+ Key []byte
+ Value []byte
+}
+
+func (h *RecordHeader) encode(pe packetEncoder) error {
+ if err := pe.putVarintBytes(h.Key); err != nil {
+ return err
+ }
+ return pe.putVarintBytes(h.Value)
+}
+
+func (h *RecordHeader) decode(pd packetDecoder) (err error) {
+ if h.Key, err = pd.getVarintBytes(); err != nil {
+ return err
+ }
+
+ if h.Value, err = pd.getVarintBytes(); err != nil {
+ return err
+ }
+ return nil
+}
+
+// Record is kafka record type
+type Record struct {
+ Headers []*RecordHeader
+
+ Attributes int8
+ TimestampDelta time.Duration
+ OffsetDelta int64
+ Key []byte
+ Value []byte
+ length varintLengthField
+}
+
+func (r *Record) encode(pe packetEncoder) error {
+ pe.push(&r.length)
+ pe.putInt8(r.Attributes)
+ pe.putVarint(int64(r.TimestampDelta / time.Millisecond))
+ pe.putVarint(r.OffsetDelta)
+ if err := pe.putVarintBytes(r.Key); err != nil {
+ return err
+ }
+ if err := pe.putVarintBytes(r.Value); err != nil {
+ return err
+ }
+ pe.putVarint(int64(len(r.Headers)))
+
+ for _, h := range r.Headers {
+ if err := h.encode(pe); err != nil {
+ return err
+ }
+ }
+
+ return pe.pop()
+}
+
+func (r *Record) decode(pd packetDecoder) (err error) {
+ if err = pd.push(&r.length); err != nil {
+ return err
+ }
+
+ if r.Attributes, err = pd.getInt8(); err != nil {
+ return err
+ }
+
+ timestamp, err := pd.getVarint()
+ if err != nil {
+ return err
+ }
+ r.TimestampDelta = time.Duration(timestamp) * time.Millisecond
+
+ if r.OffsetDelta, err = pd.getVarint(); err != nil {
+ return err
+ }
+
+ if r.Key, err = pd.getVarintBytes(); err != nil {
+ return err
+ }
+
+ if r.Value, err = pd.getVarintBytes(); err != nil {
+ return err
+ }
+
+ numHeaders, err := pd.getVarint()
+ if err != nil {
+ return err
+ }
+
+ if numHeaders >= 0 {
+ r.Headers = make([]*RecordHeader, numHeaders)
+ }
+ for i := int64(0); i < numHeaders; i++ {
+ hdr := new(RecordHeader)
+ if err := hdr.decode(pd); err != nil {
+ return err
+ }
+ r.Headers[i] = hdr
+ }
+
+ return pd.pop()
+}
diff --git a/vendor/github.com/IBM/sarama/record_batch.go b/vendor/github.com/IBM/sarama/record_batch.go
new file mode 100644
index 0000000..c422c5c
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/record_batch.go
@@ -0,0 +1,226 @@
+package sarama
+
+import (
+ "errors"
+ "fmt"
+ "time"
+)
+
+const recordBatchOverhead = 49
+
+type recordsArray []*Record
+
+func (e recordsArray) encode(pe packetEncoder) error {
+ for _, r := range e {
+ if err := r.encode(pe); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (e recordsArray) decode(pd packetDecoder) error {
+ records := make([]Record, len(e))
+ for i := range e {
+ if err := records[i].decode(pd); err != nil {
+ return err
+ }
+ e[i] = &records[i]
+ }
+ return nil
+}
+
+type RecordBatch struct {
+ FirstOffset int64
+ PartitionLeaderEpoch int32
+ Version int8
+ Codec CompressionCodec
+ CompressionLevel int
+ Control bool
+ LogAppendTime bool
+ LastOffsetDelta int32
+ FirstTimestamp time.Time
+ MaxTimestamp time.Time
+ ProducerID int64
+ ProducerEpoch int16
+ FirstSequence int32
+ Records []*Record
+ PartialTrailingRecord bool
+ IsTransactional bool
+
+ compressedRecords []byte
+ recordsLen int // uncompressed records size
+}
+
+func (b *RecordBatch) LastOffset() int64 {
+ return b.FirstOffset + int64(b.LastOffsetDelta)
+}
+
+func (b *RecordBatch) encode(pe packetEncoder) error {
+ if b.Version != 2 {
+ return PacketEncodingError{fmt.Sprintf("unsupported record batch version (%d)", b.Version)}
+ }
+ pe.putInt64(b.FirstOffset)
+ pe.push(&lengthField{})
+ pe.putInt32(b.PartitionLeaderEpoch)
+ pe.putInt8(b.Version)
+ pe.push(newCRC32Field(crcCastagnoli))
+ pe.putInt16(b.computeAttributes())
+ pe.putInt32(b.LastOffsetDelta)
+
+ if err := (Timestamp{&b.FirstTimestamp}).encode(pe); err != nil {
+ return err
+ }
+
+ if err := (Timestamp{&b.MaxTimestamp}).encode(pe); err != nil {
+ return err
+ }
+
+ pe.putInt64(b.ProducerID)
+ pe.putInt16(b.ProducerEpoch)
+ pe.putInt32(b.FirstSequence)
+
+ if err := pe.putArrayLength(len(b.Records)); err != nil {
+ return err
+ }
+
+ if b.compressedRecords == nil {
+ if err := b.encodeRecords(pe); err != nil {
+ return err
+ }
+ }
+ if err := pe.putRawBytes(b.compressedRecords); err != nil {
+ return err
+ }
+
+ if err := pe.pop(); err != nil {
+ return err
+ }
+ return pe.pop()
+}
+
+func (b *RecordBatch) decode(pd packetDecoder) (err error) {
+ if b.FirstOffset, err = pd.getInt64(); err != nil {
+ return err
+ }
+
+ batchLen, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ if b.PartitionLeaderEpoch, err = pd.getInt32(); err != nil {
+ return err
+ }
+
+ if b.Version, err = pd.getInt8(); err != nil {
+ return err
+ }
+
+ crc32Decoder := acquireCrc32Field(crcCastagnoli)
+ defer releaseCrc32Field(crc32Decoder)
+
+ if err = pd.push(crc32Decoder); err != nil {
+ return err
+ }
+
+ attributes, err := pd.getInt16()
+ if err != nil {
+ return err
+ }
+ b.Codec = CompressionCodec(int8(attributes) & compressionCodecMask)
+ b.Control = attributes&controlMask == controlMask
+ b.LogAppendTime = attributes×tampTypeMask == timestampTypeMask
+ b.IsTransactional = attributes&isTransactionalMask == isTransactionalMask
+
+ if b.LastOffsetDelta, err = pd.getInt32(); err != nil {
+ return err
+ }
+
+ if err = (Timestamp{&b.FirstTimestamp}).decode(pd); err != nil {
+ return err
+ }
+
+ if err = (Timestamp{&b.MaxTimestamp}).decode(pd); err != nil {
+ return err
+ }
+
+ if b.ProducerID, err = pd.getInt64(); err != nil {
+ return err
+ }
+
+ if b.ProducerEpoch, err = pd.getInt16(); err != nil {
+ return err
+ }
+
+ if b.FirstSequence, err = pd.getInt32(); err != nil {
+ return err
+ }
+
+ numRecs, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+ if numRecs >= 0 {
+ b.Records = make([]*Record, numRecs)
+ }
+
+ bufSize := int(batchLen) - recordBatchOverhead
+ recBuffer, err := pd.getRawBytes(bufSize)
+ if err != nil {
+ if errors.Is(err, ErrInsufficientData) {
+ b.PartialTrailingRecord = true
+ b.Records = nil
+ return nil
+ }
+ return err
+ }
+
+ if err = pd.pop(); err != nil {
+ return err
+ }
+
+ recBuffer, err = decompress(b.Codec, recBuffer)
+ if err != nil {
+ return err
+ }
+
+ b.recordsLen = len(recBuffer)
+ err = decode(recBuffer, recordsArray(b.Records), nil)
+ if errors.Is(err, ErrInsufficientData) {
+ b.PartialTrailingRecord = true
+ b.Records = nil
+ return nil
+ }
+ return err
+}
+
+func (b *RecordBatch) encodeRecords(pe packetEncoder) error {
+ var raw []byte
+ var err error
+ if raw, err = encode(recordsArray(b.Records), pe.metricRegistry()); err != nil {
+ return err
+ }
+ b.recordsLen = len(raw)
+
+ b.compressedRecords, err = compress(b.Codec, b.CompressionLevel, raw)
+ return err
+}
+
+func (b *RecordBatch) computeAttributes() int16 {
+ attr := int16(b.Codec) & int16(compressionCodecMask)
+ if b.Control {
+ attr |= controlMask
+ }
+ if b.LogAppendTime {
+ attr |= timestampTypeMask
+ }
+ if b.IsTransactional {
+ attr |= isTransactionalMask
+ }
+ return attr
+}
+
+func (b *RecordBatch) addRecord(r *Record) {
+ b.Records = append(b.Records, r)
+}
diff --git a/vendor/github.com/IBM/sarama/records.go b/vendor/github.com/IBM/sarama/records.go
new file mode 100644
index 0000000..ec03d71
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/records.go
@@ -0,0 +1,219 @@
+package sarama
+
+import "fmt"
+
+const (
+ unknownRecords = iota
+ legacyRecords
+ defaultRecords
+
+ magicOffset = 16
+)
+
+// Records implements a union type containing either a RecordBatch or a legacy MessageSet.
+type Records struct {
+ recordsType int
+ MsgSet *MessageSet
+ RecordBatch *RecordBatch
+}
+
+func newLegacyRecords(msgSet *MessageSet) Records {
+ return Records{recordsType: legacyRecords, MsgSet: msgSet}
+}
+
+func newDefaultRecords(batch *RecordBatch) Records {
+ return Records{recordsType: defaultRecords, RecordBatch: batch}
+}
+
+// setTypeFromFields sets type of Records depending on which of MsgSet or RecordBatch is not nil.
+// The first return value indicates whether both fields are nil (and the type is not set).
+// If both fields are not nil, it returns an error.
+func (r *Records) setTypeFromFields() (bool, error) {
+ if r.MsgSet == nil && r.RecordBatch == nil {
+ return true, nil
+ }
+ if r.MsgSet != nil && r.RecordBatch != nil {
+ return false, fmt.Errorf("both MsgSet and RecordBatch are set, but record type is unknown")
+ }
+ r.recordsType = defaultRecords
+ if r.MsgSet != nil {
+ r.recordsType = legacyRecords
+ }
+ return false, nil
+}
+
+func (r *Records) encode(pe packetEncoder) error {
+ if r.recordsType == unknownRecords {
+ if empty, err := r.setTypeFromFields(); err != nil || empty {
+ return err
+ }
+ }
+
+ switch r.recordsType {
+ case legacyRecords:
+ if r.MsgSet == nil {
+ return nil
+ }
+ return r.MsgSet.encode(pe)
+ case defaultRecords:
+ if r.RecordBatch == nil {
+ return nil
+ }
+ return r.RecordBatch.encode(pe)
+ }
+
+ return fmt.Errorf("unknown records type: %v", r.recordsType)
+}
+
+func (r *Records) setTypeFromMagic(pd packetDecoder) error {
+ magic, err := magicValue(pd)
+ if err != nil {
+ return err
+ }
+
+ r.recordsType = defaultRecords
+ if magic < 2 {
+ r.recordsType = legacyRecords
+ }
+
+ return nil
+}
+
+func (r *Records) decode(pd packetDecoder) error {
+ if r.recordsType == unknownRecords {
+ if err := r.setTypeFromMagic(pd); err != nil {
+ return err
+ }
+ }
+
+ switch r.recordsType {
+ case legacyRecords:
+ r.MsgSet = &MessageSet{}
+ return r.MsgSet.decode(pd)
+ case defaultRecords:
+ r.RecordBatch = &RecordBatch{}
+ return r.RecordBatch.decode(pd)
+ }
+ return fmt.Errorf("unknown records type: %v", r.recordsType)
+}
+
+func (r *Records) numRecords() (int, error) {
+ if r.recordsType == unknownRecords {
+ if empty, err := r.setTypeFromFields(); err != nil || empty {
+ return 0, err
+ }
+ }
+
+ switch r.recordsType {
+ case legacyRecords:
+ if r.MsgSet == nil {
+ return 0, nil
+ }
+ return len(r.MsgSet.Messages), nil
+ case defaultRecords:
+ if r.RecordBatch == nil {
+ return 0, nil
+ }
+ return len(r.RecordBatch.Records), nil
+ }
+ return 0, fmt.Errorf("unknown records type: %v", r.recordsType)
+}
+
+func (r *Records) isPartial() (bool, error) {
+ if r.recordsType == unknownRecords {
+ if empty, err := r.setTypeFromFields(); err != nil || empty {
+ return false, err
+ }
+ }
+
+ switch r.recordsType {
+ case unknownRecords:
+ return false, nil
+ case legacyRecords:
+ if r.MsgSet == nil {
+ return false, nil
+ }
+ return r.MsgSet.PartialTrailingMessage, nil
+ case defaultRecords:
+ if r.RecordBatch == nil {
+ return false, nil
+ }
+ return r.RecordBatch.PartialTrailingRecord, nil
+ }
+ return false, fmt.Errorf("unknown records type: %v", r.recordsType)
+}
+
+func (r *Records) isControl() (bool, error) {
+ if r.recordsType == unknownRecords {
+ if empty, err := r.setTypeFromFields(); err != nil || empty {
+ return false, err
+ }
+ }
+
+ switch r.recordsType {
+ case legacyRecords:
+ return false, nil
+ case defaultRecords:
+ if r.RecordBatch == nil {
+ return false, nil
+ }
+ return r.RecordBatch.Control, nil
+ }
+ return false, fmt.Errorf("unknown records type: %v", r.recordsType)
+}
+
+func (r *Records) isOverflow() (bool, error) {
+ if r.recordsType == unknownRecords {
+ if empty, err := r.setTypeFromFields(); err != nil || empty {
+ return false, err
+ }
+ }
+
+ switch r.recordsType {
+ case unknownRecords:
+ return false, nil
+ case legacyRecords:
+ if r.MsgSet == nil {
+ return false, nil
+ }
+ return r.MsgSet.OverflowMessage, nil
+ case defaultRecords:
+ return false, nil
+ }
+ return false, fmt.Errorf("unknown records type: %v", r.recordsType)
+}
+
+func (r *Records) nextOffset() (*int64, error) {
+ switch r.recordsType {
+ case unknownRecords:
+ return nil, nil
+ case legacyRecords:
+ return nil, nil
+ case defaultRecords:
+ if r.RecordBatch == nil {
+ return nil, nil
+ }
+ nextOffset := r.RecordBatch.LastOffset() + 1
+ return &nextOffset, nil
+ }
+ return nil, fmt.Errorf("unknown records type: %v", r.recordsType)
+}
+
+func magicValue(pd packetDecoder) (int8, error) {
+ return pd.peekInt8(magicOffset)
+}
+
+func (r *Records) getControlRecord() (ControlRecord, error) {
+ if r.RecordBatch == nil || len(r.RecordBatch.Records) == 0 {
+ return ControlRecord{}, fmt.Errorf("cannot get control record, record batch is empty")
+ }
+
+ firstRecord := r.RecordBatch.Records[0]
+ controlRecord := ControlRecord{}
+ err := controlRecord.decode(&realDecoder{raw: firstRecord.Key}, &realDecoder{raw: firstRecord.Value})
+ if err != nil {
+ return ControlRecord{}, err
+ }
+
+ return controlRecord, nil
+}
diff --git a/vendor/github.com/IBM/sarama/request.go b/vendor/github.com/IBM/sarama/request.go
new file mode 100644
index 0000000..83a1c46
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/request.go
@@ -0,0 +1,239 @@
+package sarama
+
+import (
+ "encoding/binary"
+ "fmt"
+ "io"
+)
+
+type protocolBody interface {
+ encoder
+ versionedDecoder
+ key() int16
+ version() int16
+ setVersion(int16)
+ headerVersion() int16
+ isValidVersion() bool
+ requiredVersion() KafkaVersion
+}
+
+type request struct {
+ correlationID int32
+ clientID string
+ body protocolBody
+}
+
+func (r *request) encode(pe packetEncoder) error {
+ pe.push(&lengthField{})
+ pe.putInt16(r.body.key())
+ pe.putInt16(r.body.version())
+ pe.putInt32(r.correlationID)
+
+ if r.body.headerVersion() >= 1 {
+ err := pe.putString(r.clientID)
+ if err != nil {
+ return err
+ }
+ }
+
+ if r.body.headerVersion() >= 2 {
+ // we don't use tag headers at the moment so we just put an array length of 0
+ pe.putUVarint(0)
+ }
+ pe = prepareFlexibleEncoder(pe, r.body)
+
+ err := r.body.encode(pe)
+ if err != nil {
+ return err
+ }
+
+ return pe.pop()
+}
+
+func (r *request) decode(pd packetDecoder) (err error) {
+ key, err := pd.getInt16()
+ if err != nil {
+ return err
+ }
+
+ version, err := pd.getInt16()
+ if err != nil {
+ return err
+ }
+
+ r.correlationID, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ r.clientID, err = pd.getString()
+ if err != nil {
+ return err
+ }
+
+ r.body = allocateBody(key, version)
+ if r.body == nil {
+ return PacketDecodingError{fmt.Sprintf("unknown request key (%d)", key)}
+ }
+
+ if r.body.headerVersion() >= 2 {
+ // tagged field
+ _, err = pd.getUVarint()
+ if err != nil {
+ return err
+ }
+ }
+
+ if decoder, ok := pd.(*realDecoder); ok {
+ pd = prepareFlexibleDecoder(decoder, r.body, version)
+ }
+ return r.body.decode(pd, version)
+}
+
+func decodeRequest(r io.Reader) (*request, int, error) {
+ var (
+ bytesRead int
+ lengthBytes = make([]byte, 4)
+ )
+
+ if n, err := io.ReadFull(r, lengthBytes); err != nil {
+ return nil, n, err
+ }
+
+ bytesRead += len(lengthBytes)
+ length := int32(binary.BigEndian.Uint32(lengthBytes))
+
+ if length <= 4 || length > MaxRequestSize {
+ return nil, bytesRead, PacketDecodingError{fmt.Sprintf("message of length %d too large or too small", length)}
+ }
+
+ encodedReq := make([]byte, length)
+ if n, err := io.ReadFull(r, encodedReq); err != nil {
+ return nil, bytesRead + n, err
+ }
+
+ bytesRead += len(encodedReq)
+
+ req := &request{}
+ if err := decode(encodedReq, req, nil); err != nil {
+ return nil, bytesRead, err
+ }
+
+ return req, bytesRead, nil
+}
+
+func allocateBody(key, version int16) protocolBody {
+ switch key {
+ case apiKeyProduce:
+ return &ProduceRequest{Version: version}
+ case apiKeyFetch:
+ return &FetchRequest{Version: version}
+ case apiKeyListOffsets:
+ return &OffsetRequest{Version: version}
+ case apiKeyMetadata:
+ return &MetadataRequest{Version: version}
+ // 4: LeaderAndIsrRequest
+ // 5: StopReplicaRequest
+ // 6: UpdateMetadataRequest
+ // 7: ControlledShutdownRequest
+ case apiKeyOffsetCommit:
+ return &OffsetCommitRequest{Version: version}
+ case apiKeyOffsetFetch:
+ return &OffsetFetchRequest{Version: version}
+ case apiKeyFindCoordinator:
+ return &FindCoordinatorRequest{Version: version}
+ case apiKeyJoinGroup:
+ return &JoinGroupRequest{Version: version}
+ case apiKeyHeartbeat:
+ return &HeartbeatRequest{Version: version}
+ case apiKeyLeaveGroup:
+ return &LeaveGroupRequest{Version: version}
+ case apiKeySyncGroup:
+ return &SyncGroupRequest{Version: version}
+ case apiKeyDescribeGroups:
+ return &DescribeGroupsRequest{Version: version}
+ case apiKeyListGroups:
+ return &ListGroupsRequest{Version: version}
+ case apiKeySaslHandshake:
+ return &SaslHandshakeRequest{Version: version}
+ case apiKeyApiVersions:
+ return &ApiVersionsRequest{Version: version}
+ case apiKeyCreateTopics:
+ return &CreateTopicsRequest{Version: version}
+ case apiKeyDeleteTopics:
+ return &DeleteTopicsRequest{Version: version}
+ case apiKeyDeleteRecords:
+ return &DeleteRecordsRequest{Version: version}
+ case apiKeyInitProducerId:
+ return &InitProducerIDRequest{Version: version}
+ // 23: OffsetForLeaderEpochRequest
+ case apiKeyAddPartitionsToTxn:
+ return &AddPartitionsToTxnRequest{Version: version}
+ case apiKeyAddOffsetsToTxn:
+ return &AddOffsetsToTxnRequest{Version: version}
+ case apiKeyEndTxn:
+ return &EndTxnRequest{Version: version}
+ // 27: WriteTxnMarkersRequest
+ case apiKeyTxnOffsetCommit:
+ return &TxnOffsetCommitRequest{Version: version}
+ case apiKeyDescribeAcls:
+ return &DescribeAclsRequest{Version: int(version)}
+ case apiKeyCreateAcls:
+ return &CreateAclsRequest{Version: version}
+ case apiKeyDeleteAcls:
+ return &DeleteAclsRequest{Version: int(version)}
+ case apiKeyDescribeConfigs:
+ return &DescribeConfigsRequest{Version: version}
+ case apiKeyAlterConfigs:
+ return &AlterConfigsRequest{Version: version}
+ // 34: AlterReplicaLogDirsRequest
+ case apiKeyDescribeLogDirs:
+ return &DescribeLogDirsRequest{Version: version}
+ case apiKeySASLAuth:
+ return &SaslAuthenticateRequest{Version: version}
+ case apiKeyCreatePartitions:
+ return &CreatePartitionsRequest{Version: version}
+ // 38: CreateDelegationTokenRequest
+ // 39: RenewDelegationTokenRequest
+ // 40: ExpireDelegationTokenRequest
+ // 41: DescribeDelegationTokenRequest
+ case apiKeyDeleteGroups:
+ return &DeleteGroupsRequest{Version: version}
+ case apiKeyElectLeaders:
+ return &ElectLeadersRequest{Version: version}
+ case apiKeyIncrementalAlterConfigs:
+ return &IncrementalAlterConfigsRequest{Version: version}
+ case apiKeyAlterPartitionReassignments:
+ return &AlterPartitionReassignmentsRequest{Version: version}
+ case apiKeyListPartitionReassignments:
+ return &ListPartitionReassignmentsRequest{Version: version}
+ case apiKeyOffsetDelete:
+ return &DeleteOffsetsRequest{Version: version}
+ case apiKeyDescribeClientQuotas:
+ return &DescribeClientQuotasRequest{Version: version}
+ case apiKeyAlterClientQuotas:
+ return &AlterClientQuotasRequest{Version: version}
+ case apiKeyDescribeUserScramCredentials:
+ return &DescribeUserScramCredentialsRequest{Version: version}
+ case apiKeyAlterUserScramCredentials:
+ return &AlterUserScramCredentialsRequest{Version: version}
+ // 52: VoteRequest
+ // 53: BeginQuorumEpochRequest
+ // 54: EndQuorumEpochRequest
+ // 55: DescribeQuorumRequest
+ // 56: AlterPartitionRequest
+ // 57: UpdateFeaturesRequest
+ // 58: EnvelopeRequest
+ // 59: FetchSnapshotRequest
+ // 60: DescribeClusterRequest
+ // 61: DescribeProducersRequest
+ // 62: BrokerRegistrationRequest
+ // 63: BrokerHeartbeatRequest
+ // 64: UnregisterBrokerRequest
+ // 65: DescribeTransactionsRequest
+ // 66: ListTransactionsRequest
+ // 67: AllocateProducerIdsRequest
+ // 68: ConsumerGroupHeartbeatRequest
+ }
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/response_header.go b/vendor/github.com/IBM/sarama/response_header.go
new file mode 100644
index 0000000..88b0b44
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/response_header.go
@@ -0,0 +1,33 @@
+package sarama
+
+import "fmt"
+
+type responseHeader struct {
+ length int32
+ correlationID int32
+}
+
+func (r *responseHeader) decode(pd packetDecoder, version int16) (err error) {
+ if version >= 1 {
+ if decoder, ok := pd.(*realDecoder); ok {
+ pd = &realFlexibleDecoder{decoder}
+ } else {
+ return PacketDecodingError{"failed to instantiate flexible decoder"}
+ }
+ }
+ r.length, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+ if r.length <= 4 || r.length > MaxResponseSize {
+ return PacketDecodingError{fmt.Sprintf("message of length %d too large or too small", r.length)}
+ }
+
+ r.correlationID, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
diff --git a/vendor/github.com/IBM/sarama/sarama.go b/vendor/github.com/IBM/sarama/sarama.go
new file mode 100644
index 0000000..4d5f60a
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/sarama.go
@@ -0,0 +1,139 @@
+/*
+Package sarama is a pure Go client library for dealing with Apache Kafka (versions 0.8 and later). It includes a high-level
+API for easily producing and consuming messages, and a low-level API for controlling bytes on the wire when the high-level
+API is insufficient. Usage examples for the high-level APIs are provided inline with their full documentation.
+
+To produce messages, use either the AsyncProducer or the SyncProducer. The AsyncProducer accepts messages on a channel
+and produces them asynchronously in the background as efficiently as possible; it is preferred in most cases.
+The SyncProducer provides a method which will block until Kafka acknowledges the message as produced. This can be
+useful but comes with two caveats: it will generally be less efficient, and the actual durability guarantees
+depend on the configured value of `Producer.RequiredAcks`. There are configurations where a message acknowledged by the
+SyncProducer can still sometimes be lost.
+
+To consume messages, use Consumer or Consumer-Group API.
+
+For lower-level needs, the Broker and Request/Response objects permit precise control over each connection
+and message sent on the wire; the Client provides higher-level metadata management that is shared between
+the producers and the consumer. The Request/Response objects and properties are mostly undocumented, as they line up
+exactly with the protocol fields documented by Kafka at
+https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol
+
+Metrics are exposed through https://github.com/rcrowley/go-metrics library in a local registry.
+
+Broker related metrics:
+
+ +---------------------------------------------------------+------------+---------------------------------------------------------------+
+ | Name | Type | Description |
+ +---------------------------------------------------------+------------+---------------------------------------------------------------+
+ | incoming-byte-rate | meter | Bytes/second read off all brokers |
+ | incoming-byte-rate-for-broker-<broker-id> | meter | Bytes/second read off a given broker |
+ | outgoing-byte-rate | meter | Bytes/second written off all brokers |
+ | outgoing-byte-rate-for-broker-<broker-id> | meter | Bytes/second written off a given broker |
+ | request-rate | meter | Requests/second sent to all brokers |
+ | request-rate-for-broker-<broker-id> | meter | Requests/second sent to a given broker |
+ | request-size | histogram | Distribution of the request size in bytes for all brokers |
+ | request-size-for-broker-<broker-id> | histogram | Distribution of the request size in bytes for a given broker |
+ | request-latency-in-ms | histogram | Distribution of the request latency in ms for all brokers |
+ | request-latency-in-ms-for-broker-<broker-id> | histogram | Distribution of the request latency in ms for a given broker |
+ | response-rate | meter | Responses/second received from all brokers |
+ | response-rate-for-broker-<broker-id> | meter | Responses/second received from a given broker |
+ | response-size | histogram | Distribution of the response size in bytes for all brokers |
+ | response-size-for-broker-<broker-id> | histogram | Distribution of the response size in bytes for a given broker |
+ | requests-in-flight | counter | The current number of in-flight requests awaiting a response |
+ | | | for all brokers |
+ | requests-in-flight-for-broker-<broker-id> | counter | The current number of in-flight requests awaiting a response |
+ | | | for a given broker |
+ | protocol-requests-rate-<api-key> | meter | Number of api requests sent to the brokers for all brokers |
+ | | | https://kafka.apache.org/protocol.html#protocol_api_keys | |
+ | protocol-requests-rate-<api-key>-for-broker-<broker-id> | meter | Number of packets sent to the brokers by api-key for a given |
+ | | | broker |
+ +---------------------------------------------------------+------------+---------------------------------------------------------------+
+
+Note that we do not gather specific metrics for seed brokers but they are part of the "all brokers" metrics.
+
+Producer related metrics:
+
+ +-------------------------------------------+------------+--------------------------------------------------------------------------------------+
+ | Name | Type | Description |
+ +-------------------------------------------+------------+--------------------------------------------------------------------------------------+
+ | batch-size | histogram | Distribution of the number of bytes sent per partition per request for all topics |
+ | batch-size-for-topic-<topic> | histogram | Distribution of the number of bytes sent per partition per request for a given topic |
+ | record-send-rate | meter | Records/second sent to all topics |
+ | record-send-rate-for-topic-<topic> | meter | Records/second sent to a given topic |
+ | records-per-request | histogram | Distribution of the number of records sent per request for all topics |
+ | records-per-request-for-topic-<topic> | histogram | Distribution of the number of records sent per request for a given topic |
+ | compression-ratio | histogram | Distribution of the compression ratio times 100 of record batches for all topics |
+ | compression-ratio-for-topic-<topic> | histogram | Distribution of the compression ratio times 100 of record batches for a given topic |
+ +-------------------------------------------+------------+--------------------------------------------------------------------------------------+
+
+Consumer related metrics:
+
+ +-------------------------------------------+------------+--------------------------------------------------------------------------------------+
+ | Name | Type | Description |
+ +-------------------------------------------+------------+--------------------------------------------------------------------------------------+
+ | consumer-batch-size | histogram | Distribution of the number of messages in a batch |
+ | consumer-fetch-rate | meter | Fetch requests/second sent to all brokers |
+ | consumer-fetch-rate-for-broker-<broker> | meter | Fetch requests/second sent to a given broker |
+ | consumer-fetch-rate-for-topic-<topic> | meter | Fetch requests/second sent for a given topic |
+ | consumer-fetch-response-size | histogram | Distribution of the fetch response size in bytes |
+ | consumer-group-join-total-<GroupID> | counter | Total count of consumer group join attempts |
+ | consumer-group-join-failed-<GroupID> | counter | Total count of consumer group join failures |
+ | consumer-group-sync-total-<GroupID> | counter | Total count of consumer group sync attempts |
+ | consumer-group-sync-failed-<GroupID> | counter | Total count of consumer group sync failures |
+ +-------------------------------------------+------------+--------------------------------------------------------------------------------------+
+*/
+package sarama
+
+import (
+ "io"
+ "log"
+)
+
+var (
+ // Logger is the instance of a StdLogger interface that Sarama writes connection
+ // management events to. By default it is set to discard all log messages via io.Discard,
+ // but you can set it to redirect wherever you want.
+ Logger StdLogger = log.New(io.Discard, "[Sarama] ", log.LstdFlags)
+
+ // PanicHandler is called for recovering from panics spawned internally to the library (and thus
+ // not recoverable by the caller's goroutine). Defaults to nil, which means panics are not recovered.
+ PanicHandler func(interface{})
+
+ // MaxRequestSize is the maximum size (in bytes) of any request that Sarama will attempt to send. Trying
+ // to send a request larger than this will result in an PacketEncodingError. The default of 100 MiB is aligned
+ // with Kafka's default `socket.request.max.bytes`, which is the largest request the broker will attempt
+ // to process.
+ MaxRequestSize int32 = 100 * 1024 * 1024
+
+ // MaxResponseSize is the maximum size (in bytes) of any response that Sarama will attempt to parse. If
+ // a broker returns a response message larger than this value, Sarama will return a PacketDecodingError to
+ // protect the client from running out of memory. Please note that brokers do not have any natural limit on
+ // the size of responses they send. In particular, they can send arbitrarily large fetch responses to consumers
+ // (see https://issues.apache.org/jira/browse/KAFKA-2063).
+ MaxResponseSize int32 = 100 * 1024 * 1024
+)
+
+// StdLogger is used to log error messages.
+type StdLogger interface {
+ Print(v ...interface{})
+ Printf(format string, v ...interface{})
+ Println(v ...interface{})
+}
+
+type debugLogger struct{}
+
+func (d *debugLogger) Print(v ...interface{}) {
+ Logger.Print(v...)
+}
+func (d *debugLogger) Printf(format string, v ...interface{}) {
+ Logger.Printf(format, v...)
+}
+func (d *debugLogger) Println(v ...interface{}) {
+ Logger.Println(v...)
+}
+
+// DebugLogger is the instance of a StdLogger that Sarama writes more verbose
+// debug information to. By default it is set to redirect all debug to the
+// default Logger above, but you can optionally set it to another StdLogger
+// instance to (e.g.,) discard debug information
+var DebugLogger StdLogger = &debugLogger{}
diff --git a/vendor/github.com/IBM/sarama/sasl_authenticate_request.go b/vendor/github.com/IBM/sarama/sasl_authenticate_request.go
new file mode 100644
index 0000000..f53a991
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/sasl_authenticate_request.go
@@ -0,0 +1,46 @@
+package sarama
+
+type SaslAuthenticateRequest struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ SaslAuthBytes []byte
+}
+
+func (r *SaslAuthenticateRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *SaslAuthenticateRequest) encode(pe packetEncoder) error {
+ return pe.putBytes(r.SaslAuthBytes)
+}
+
+func (r *SaslAuthenticateRequest) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ r.SaslAuthBytes, err = pd.getBytes()
+ return err
+}
+
+func (r *SaslAuthenticateRequest) key() int16 {
+ return apiKeySASLAuth
+}
+
+func (r *SaslAuthenticateRequest) version() int16 {
+ return r.Version
+}
+
+func (r *SaslAuthenticateRequest) headerVersion() int16 {
+ return 1
+}
+
+func (r *SaslAuthenticateRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 1
+}
+
+func (r *SaslAuthenticateRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 1:
+ return V2_2_0_0
+ default:
+ return V1_0_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/sasl_authenticate_response.go b/vendor/github.com/IBM/sarama/sasl_authenticate_response.go
new file mode 100644
index 0000000..60752e8
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/sasl_authenticate_response.go
@@ -0,0 +1,75 @@
+package sarama
+
+type SaslAuthenticateResponse struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ Err KError
+ ErrorMessage *string
+ SaslAuthBytes []byte
+ SessionLifetimeMs int64
+}
+
+func (r *SaslAuthenticateResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *SaslAuthenticateResponse) encode(pe packetEncoder) error {
+ pe.putKError(r.Err)
+ if err := pe.putNullableString(r.ErrorMessage); err != nil {
+ return err
+ }
+ if err := pe.putBytes(r.SaslAuthBytes); err != nil {
+ return err
+ }
+ if r.Version > 0 {
+ pe.putInt64(r.SessionLifetimeMs)
+ }
+ return nil
+}
+
+func (r *SaslAuthenticateResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ r.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if r.ErrorMessage, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ if r.SaslAuthBytes, err = pd.getBytes(); err != nil {
+ return err
+ }
+
+ if version > 0 {
+ r.SessionLifetimeMs, err = pd.getInt64()
+ }
+
+ return err
+}
+
+func (r *SaslAuthenticateResponse) key() int16 {
+ return apiKeySASLAuth
+}
+
+func (r *SaslAuthenticateResponse) version() int16 {
+ return r.Version
+}
+
+func (r *SaslAuthenticateResponse) headerVersion() int16 {
+ return 0
+}
+
+func (r *SaslAuthenticateResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 1
+}
+
+func (r *SaslAuthenticateResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 1:
+ return V2_2_0_0
+ default:
+ return V1_0_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/sasl_handshake_request.go b/vendor/github.com/IBM/sarama/sasl_handshake_request.go
new file mode 100644
index 0000000..cae8536
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/sasl_handshake_request.go
@@ -0,0 +1,51 @@
+package sarama
+
+type SaslHandshakeRequest struct {
+ Mechanism string
+ Version int16
+}
+
+func (r *SaslHandshakeRequest) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *SaslHandshakeRequest) encode(pe packetEncoder) error {
+ if err := pe.putString(r.Mechanism); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (r *SaslHandshakeRequest) decode(pd packetDecoder, version int16) (err error) {
+ if r.Mechanism, err = pd.getString(); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (r *SaslHandshakeRequest) key() int16 {
+ return apiKeySaslHandshake
+}
+
+func (r *SaslHandshakeRequest) version() int16 {
+ return r.Version
+}
+
+func (r *SaslHandshakeRequest) headerVersion() int16 {
+ return 1
+}
+
+func (r *SaslHandshakeRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 1
+}
+
+func (r *SaslHandshakeRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 1:
+ return V1_0_0_0
+ default:
+ return V0_10_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/sasl_handshake_response.go b/vendor/github.com/IBM/sarama/sasl_handshake_response.go
new file mode 100644
index 0000000..3974589
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/sasl_handshake_response.go
@@ -0,0 +1,54 @@
+package sarama
+
+type SaslHandshakeResponse struct {
+ Version int16
+ Err KError
+ EnabledMechanisms []string
+}
+
+func (r *SaslHandshakeResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *SaslHandshakeResponse) encode(pe packetEncoder) error {
+ pe.putKError(r.Err)
+ return pe.putStringArray(r.EnabledMechanisms)
+}
+
+func (r *SaslHandshakeResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ if r.EnabledMechanisms, err = pd.getStringArray(); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (r *SaslHandshakeResponse) key() int16 {
+ return apiKeySaslHandshake
+}
+
+func (r *SaslHandshakeResponse) version() int16 {
+ return r.Version
+}
+
+func (r *SaslHandshakeResponse) headerVersion() int16 {
+ return 0
+}
+
+func (r *SaslHandshakeResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 1
+}
+
+func (r *SaslHandshakeResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 1:
+ return V1_0_0_0
+ default:
+ return V0_10_0_0
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/scram_formatter.go b/vendor/github.com/IBM/sarama/scram_formatter.go
new file mode 100644
index 0000000..2af9e4a
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/scram_formatter.go
@@ -0,0 +1,78 @@
+package sarama
+
+import (
+ "crypto/hmac"
+ "crypto/sha256"
+ "crypto/sha512"
+ "hash"
+)
+
+// ScramFormatter implementation
+// @see: https://github.com/apache/kafka/blob/99b9b3e84f4e98c3f07714e1de6a139a004cbc5b/clients/src/main/java/org/apache/kafka/common/security/scram/internals/ScramFormatter.java#L93
+type scramFormatter struct {
+ mechanism ScramMechanismType
+}
+
+func (s scramFormatter) mac(key []byte) (hash.Hash, error) {
+ var m hash.Hash
+
+ switch s.mechanism {
+ case SCRAM_MECHANISM_SHA_256:
+ m = hmac.New(sha256.New, key)
+
+ case SCRAM_MECHANISM_SHA_512:
+ m = hmac.New(sha512.New, key)
+ default:
+ return nil, ErrUnknownScramMechanism
+ }
+
+ return m, nil
+}
+
+func (s scramFormatter) hmac(key []byte, extra []byte) ([]byte, error) {
+ mac, err := s.mac(key)
+ if err != nil {
+ return nil, err
+ }
+
+ if _, err := mac.Write(extra); err != nil {
+ return nil, err
+ }
+ return mac.Sum(nil), nil
+}
+
+func (s scramFormatter) xor(result []byte, second []byte) {
+ for i := 0; i < len(result); i++ {
+ result[i] = result[i] ^ second[i]
+ }
+}
+
+func (s scramFormatter) saltedPassword(password []byte, salt []byte, iterations int) ([]byte, error) {
+ mac, err := s.mac(password)
+ if err != nil {
+ return nil, err
+ }
+
+ if _, err := mac.Write(salt); err != nil {
+ return nil, err
+ }
+ if _, err := mac.Write([]byte{0, 0, 0, 1}); err != nil {
+ return nil, err
+ }
+
+ u1 := mac.Sum(nil)
+ prev := u1
+ result := u1
+
+ for i := 2; i <= iterations; i++ {
+ ui, err := s.hmac(password, prev)
+ if err != nil {
+ return nil, err
+ }
+
+ s.xor(result, ui)
+ prev = ui
+ }
+
+ return result, nil
+}
diff --git a/vendor/github.com/IBM/sarama/server.properties b/vendor/github.com/IBM/sarama/server.properties
new file mode 100644
index 0000000..21ba1c7
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/server.properties
@@ -0,0 +1,138 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This configuration file is intended for use in ZK-based mode, where Apache ZooKeeper is required.
+# See kafka.server.KafkaConfig for additional details and defaults
+#
+
+############################# Server Basics #############################
+
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id=0
+
+############################# Socket Server Settings #############################
+
+# The address the socket server listens on. If not configured, the host name will be equal to the value of
+# java.net.InetAddress.getCanonicalHostName(), with PLAINTEXT listener name, and port 9092.
+# FORMAT:
+# listeners = listener_name://host_name:port
+# EXAMPLE:
+# listeners = PLAINTEXT://your.host.name:9092
+#listeners=PLAINTEXT://:9092
+
+# Listener name, hostname and port the broker will advertise to clients.
+# If not set, it uses the value for "listeners".
+#advertised.listeners=PLAINTEXT://your.host.name:9092
+
+# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
+#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
+
+# The number of threads that the server uses for receiving requests from the network and sending responses to the network
+num.network.threads=3
+
+# The number of threads that the server uses for processing requests, which may include disk I/O
+num.io.threads=8
+
+# The send buffer (SO_SNDBUF) used by the socket server
+socket.send.buffer.bytes=102400
+
+# The receive buffer (SO_RCVBUF) used by the socket server
+socket.receive.buffer.bytes=102400
+
+# The maximum size of a request that the socket server will accept (protection against OOM)
+socket.request.max.bytes=104857600
+
+
+############################# Log Basics #############################
+
+# A comma separated list of directories under which to store log files
+log.dirs=/tmp/kafka-logs
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions=1
+
+# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
+# This value is recommended to be increased for installations with data dirs located in RAID array.
+num.recovery.threads.per.data.dir=1
+
+############################# Internal Topic Settings #############################
+# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
+# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
+offsets.topic.replication.factor=1
+transaction.state.log.replication.factor=1
+transaction.state.log.min.isr=1
+
+############################# Log Flush Policy #############################
+
+# Messages are immediately written to the filesystem but by default we only fsync() to sync
+# the OS cache lazily. The following configurations control the flush of data to disk.
+# There are a few important trade-offs here:
+# 1. Durability: Unflushed data may be lost if you are not using replication.
+# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
+# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.
+# The settings below allow one to configure the flush policy to flush data after a period of time or
+# every N messages (or both). This can be done globally and overridden on a per-topic basis.
+
+# The number of messages to accept before forcing a flush of data to disk
+#log.flush.interval.messages=10000
+
+# The maximum amount of time a message can sit in a log before we force a flush
+#log.flush.interval.ms=1000
+
+############################# Log Retention Policy #############################
+
+# The following configurations control the disposal of log segments. The policy can
+# be set to delete segments after a period of time, or after a given size has accumulated.
+# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
+# from the end of the log.
+
+# The minimum age of a log file to be eligible for deletion due to age
+log.retention.hours=168
+
+# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
+# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
+#log.retention.bytes=1073741824
+
+# The maximum size of a log segment file. When this size is reached a new log segment will be created.
+#log.segment.bytes=1073741824
+
+# The interval at which log segments are checked to see if they can be deleted according
+# to the retention policies
+log.retention.check.interval.ms=300000
+
+############################# Zookeeper #############################
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect=localhost:2181
+
+# Timeout in ms for connecting to zookeeper
+zookeeper.connection.timeout.ms=18000
+
+
+############################# Group Coordinator Settings #############################
+
+# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance.
+# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms.
+# The default value for this is 3 seconds.
+# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
+# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
+group.initial.rebalance.delay.ms=0
diff --git a/vendor/github.com/IBM/sarama/sticky_assignor_user_data.go b/vendor/github.com/IBM/sarama/sticky_assignor_user_data.go
new file mode 100644
index 0000000..161233f
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/sticky_assignor_user_data.go
@@ -0,0 +1,124 @@
+package sarama
+
+type topicPartitionAssignment struct {
+ Topic string
+ Partition int32
+}
+
+type StickyAssignorUserData interface {
+ partitions() []topicPartitionAssignment
+ hasGeneration() bool
+ generation() int
+}
+
+// StickyAssignorUserDataV0 holds topic partition information for an assignment
+type StickyAssignorUserDataV0 struct {
+ Topics map[string][]int32
+
+ topicPartitions []topicPartitionAssignment
+}
+
+func (m *StickyAssignorUserDataV0) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(m.Topics)); err != nil {
+ return err
+ }
+
+ for topic, partitions := range m.Topics {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putInt32Array(partitions); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (m *StickyAssignorUserDataV0) decode(pd packetDecoder) (err error) {
+ var topicLen int
+ if topicLen, err = pd.getArrayLength(); err != nil {
+ return
+ }
+
+ m.Topics = make(map[string][]int32, topicLen)
+ for i := 0; i < topicLen; i++ {
+ var topic string
+ if topic, err = pd.getString(); err != nil {
+ return
+ }
+ if m.Topics[topic], err = pd.getInt32Array(); err != nil {
+ return
+ }
+ }
+ m.topicPartitions = populateTopicPartitions(m.Topics)
+ return nil
+}
+
+func (m *StickyAssignorUserDataV0) partitions() []topicPartitionAssignment { return m.topicPartitions }
+func (m *StickyAssignorUserDataV0) hasGeneration() bool { return false }
+func (m *StickyAssignorUserDataV0) generation() int { return defaultGeneration }
+
+// StickyAssignorUserDataV1 holds topic partition information for an assignment
+type StickyAssignorUserDataV1 struct {
+ Topics map[string][]int32
+ Generation int32
+
+ topicPartitions []topicPartitionAssignment
+}
+
+func (m *StickyAssignorUserDataV1) encode(pe packetEncoder) error {
+ if err := pe.putArrayLength(len(m.Topics)); err != nil {
+ return err
+ }
+
+ for topic, partitions := range m.Topics {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putInt32Array(partitions); err != nil {
+ return err
+ }
+ }
+
+ pe.putInt32(m.Generation)
+ return nil
+}
+
+func (m *StickyAssignorUserDataV1) decode(pd packetDecoder) (err error) {
+ var topicLen int
+ if topicLen, err = pd.getArrayLength(); err != nil {
+ return
+ }
+
+ m.Topics = make(map[string][]int32, topicLen)
+ for i := 0; i < topicLen; i++ {
+ var topic string
+ if topic, err = pd.getString(); err != nil {
+ return
+ }
+ if m.Topics[topic], err = pd.getInt32Array(); err != nil {
+ return
+ }
+ }
+
+ m.Generation, err = pd.getInt32()
+ if err != nil {
+ return err
+ }
+ m.topicPartitions = populateTopicPartitions(m.Topics)
+ return nil
+}
+
+func (m *StickyAssignorUserDataV1) partitions() []topicPartitionAssignment { return m.topicPartitions }
+func (m *StickyAssignorUserDataV1) hasGeneration() bool { return true }
+func (m *StickyAssignorUserDataV1) generation() int { return int(m.Generation) }
+
+func populateTopicPartitions(topics map[string][]int32) []topicPartitionAssignment {
+ topicPartitions := make([]topicPartitionAssignment, 0)
+ for topic, partitions := range topics {
+ for _, partition := range partitions {
+ topicPartitions = append(topicPartitions, topicPartitionAssignment{Topic: topic, Partition: partition})
+ }
+ }
+ return topicPartitions
+}
diff --git a/vendor/github.com/IBM/sarama/sync_group_request.go b/vendor/github.com/IBM/sarama/sync_group_request.go
new file mode 100644
index 0000000..b109cd9
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/sync_group_request.go
@@ -0,0 +1,184 @@
+package sarama
+
+type SyncGroupRequestAssignment struct {
+ // MemberId contains the ID of the member to assign.
+ MemberId string
+ // Assignment contains the member assignment.
+ Assignment []byte
+}
+
+func (a *SyncGroupRequestAssignment) encode(pe packetEncoder, version int16) (err error) {
+ if err := pe.putString(a.MemberId); err != nil {
+ return err
+ }
+
+ if err := pe.putBytes(a.Assignment); err != nil {
+ return err
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (a *SyncGroupRequestAssignment) decode(pd packetDecoder, version int16) (err error) {
+ if a.MemberId, err = pd.getString(); err != nil {
+ return err
+ }
+
+ if a.Assignment, err = pd.getBytes(); err != nil {
+ return err
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+type SyncGroupRequest struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // GroupId contains the unique group identifier.
+ GroupId string
+ // GenerationId contains the generation of the group.
+ GenerationId int32
+ // MemberId contains the member ID assigned by the group.
+ MemberId string
+ // GroupInstanceId contains the unique identifier of the consumer instance provided by end user.
+ GroupInstanceId *string
+ // GroupAssignments contains each assignment.
+ GroupAssignments []SyncGroupRequestAssignment
+}
+
+func (s *SyncGroupRequest) setVersion(v int16) {
+ s.Version = v
+}
+
+func (s *SyncGroupRequest) encode(pe packetEncoder) (err error) {
+ if err := pe.putString(s.GroupId); err != nil {
+ return err
+ }
+
+ pe.putInt32(s.GenerationId)
+
+ if err := pe.putString(s.MemberId); err != nil {
+ return err
+ }
+
+ if s.Version >= 3 {
+ if err := pe.putNullableString(s.GroupInstanceId); err != nil {
+ return err
+ }
+ }
+
+ if err := pe.putArrayLength(len(s.GroupAssignments)); err != nil {
+ return err
+ }
+ for _, block := range s.GroupAssignments {
+ if err := block.encode(pe, s.Version); err != nil {
+ return err
+ }
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (s *SyncGroupRequest) decode(pd packetDecoder, version int16) (err error) {
+ s.Version = version
+ if s.GroupId, err = pd.getString(); err != nil {
+ return err
+ }
+
+ if s.GenerationId, err = pd.getInt32(); err != nil {
+ return err
+ }
+
+ if s.MemberId, err = pd.getString(); err != nil {
+ return err
+ }
+
+ if s.Version >= 3 {
+ if s.GroupInstanceId, err = pd.getNullableString(); err != nil {
+ return err
+ }
+ }
+
+ if numAssignments, err := pd.getArrayLength(); err != nil {
+ return err
+ } else if numAssignments > 0 {
+ s.GroupAssignments = make([]SyncGroupRequestAssignment, numAssignments)
+ for i := 0; i < numAssignments; i++ {
+ var block SyncGroupRequestAssignment
+ if err := block.decode(pd, s.Version); err != nil {
+ return err
+ }
+ s.GroupAssignments[i] = block
+ }
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *SyncGroupRequest) key() int16 {
+ return apiKeySyncGroup
+}
+
+func (r *SyncGroupRequest) version() int16 {
+ return r.Version
+}
+
+func (r *SyncGroupRequest) headerVersion() int16 {
+ if r.Version >= 4 {
+ return 2
+ }
+ return 1
+}
+
+func (r *SyncGroupRequest) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 4
+}
+
+func (r *SyncGroupRequest) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *SyncGroupRequest) isFlexibleVersion(version int16) bool {
+ return version >= 4
+}
+
+func (r *SyncGroupRequest) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 4:
+ return V2_4_0_0
+ case 3:
+ return V2_3_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_9_0_0
+ default:
+ return V2_3_0_0
+ }
+}
+
+func (r *SyncGroupRequest) AddGroupAssignment(memberId string, memberAssignment []byte) {
+ r.GroupAssignments = append(r.GroupAssignments, SyncGroupRequestAssignment{
+ MemberId: memberId,
+ Assignment: memberAssignment,
+ })
+}
+
+func (r *SyncGroupRequest) AddGroupAssignmentMember(
+ memberId string,
+ memberAssignment *ConsumerGroupMemberAssignment,
+) error {
+ bin, err := encode(memberAssignment, nil)
+ if err != nil {
+ return err
+ }
+
+ r.AddGroupAssignment(memberId, bin)
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/sync_group_response.go b/vendor/github.com/IBM/sarama/sync_group_response.go
new file mode 100644
index 0000000..a605ced
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/sync_group_response.go
@@ -0,0 +1,108 @@
+package sarama
+
+import "time"
+
+type SyncGroupResponse struct {
+ // Version defines the protocol version to use for encode and decode
+ Version int16
+ // ThrottleTime contains the duration in milliseconds for which the
+ // request was throttled due to a quota violation, or zero if the request
+ // did not violate any quota.
+ ThrottleTime int32
+ // Err contains the error code, or 0 if there was no error.
+ Err KError
+ // MemberAssignment contains the member assignment.
+ MemberAssignment []byte
+}
+
+func (r *SyncGroupResponse) setVersion(v int16) {
+ r.Version = v
+}
+
+func (r *SyncGroupResponse) GetMemberAssignment() (*ConsumerGroupMemberAssignment, error) {
+ assignment := new(ConsumerGroupMemberAssignment)
+ err := decode(r.MemberAssignment, assignment, nil)
+ return assignment, err
+}
+
+func (r *SyncGroupResponse) encode(pe packetEncoder) error {
+ if r.Version >= 1 {
+ pe.putInt32(r.ThrottleTime)
+ }
+ pe.putKError(r.Err)
+ if err := pe.putBytes(r.MemberAssignment); err != nil {
+ return err
+ }
+
+ pe.putEmptyTaggedFieldArray()
+ return nil
+}
+
+func (r *SyncGroupResponse) decode(pd packetDecoder, version int16) (err error) {
+ r.Version = version
+ if r.Version >= 1 {
+ if r.ThrottleTime, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+ r.Err, err = pd.getKError()
+ if err != nil {
+ return err
+ }
+
+ r.MemberAssignment, err = pd.getBytes()
+ if err != nil {
+ return err
+ }
+
+ _, err = pd.getEmptyTaggedFieldArray()
+ return err
+}
+
+func (r *SyncGroupResponse) key() int16 {
+ return apiKeySyncGroup
+}
+
+func (r *SyncGroupResponse) version() int16 {
+ return r.Version
+}
+
+func (r *SyncGroupResponse) headerVersion() int16 {
+ if r.Version >= 4 {
+ return 1
+ }
+ return 0
+}
+
+func (r *SyncGroupResponse) isValidVersion() bool {
+ return r.Version >= 0 && r.Version <= 4
+}
+
+func (r *SyncGroupResponse) isFlexible() bool {
+ return r.isFlexibleVersion(r.Version)
+}
+
+func (r *SyncGroupResponse) isFlexibleVersion(version int16) bool {
+ return version >= 4
+}
+
+func (r *SyncGroupResponse) requiredVersion() KafkaVersion {
+ switch r.Version {
+ case 4:
+ return V2_4_0_0
+ case 3:
+ return V2_3_0_0
+ case 2:
+ return V2_0_0_0
+ case 1:
+ return V0_11_0_0
+ case 0:
+ return V0_9_0_0
+ default:
+ return V2_3_0_0
+ }
+}
+
+func (r *SyncGroupResponse) throttleTime() time.Duration {
+ return time.Duration(r.ThrottleTime) * time.Millisecond
+}
diff --git a/vendor/github.com/IBM/sarama/sync_producer.go b/vendor/github.com/IBM/sarama/sync_producer.go
new file mode 100644
index 0000000..f6876fb
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/sync_producer.go
@@ -0,0 +1,209 @@
+package sarama
+
+import "sync"
+
+var expectationsPool = sync.Pool{
+ New: func() interface{} {
+ return make(chan *ProducerError, 1)
+ },
+}
+
+// SyncProducer publishes Kafka messages, blocking until they have been acknowledged. It routes messages to the correct
+// broker, refreshing metadata as appropriate, and parses responses for errors. You must call Close() on a producer
+// to avoid leaks, it may not be garbage-collected automatically when it passes out of scope.
+//
+// The SyncProducer comes with two caveats: it will generally be less efficient than the AsyncProducer, and the actual
+// durability guarantee provided when a message is acknowledged depend on the configured value of `Producer.RequiredAcks`.
+// There are configurations where a message acknowledged by the SyncProducer can still sometimes be lost.
+//
+// For implementation reasons, the SyncProducer requires `Producer.Return.Errors` and `Producer.Return.Successes` to
+// be set to true in its configuration.
+type SyncProducer interface {
+
+ // SendMessage produces a given message, and returns only when it either has
+ // succeeded or failed to produce. It will return the partition and the offset
+ // of the produced message, or an error if the message failed to produce.
+ SendMessage(msg *ProducerMessage) (partition int32, offset int64, err error)
+
+ // SendMessages produces a given set of messages, and returns only when all
+ // messages in the set have either succeeded or failed. Note that messages
+ // can succeed and fail individually; if some succeed and some fail,
+ // SendMessages will return an error.
+ SendMessages(msgs []*ProducerMessage) error
+
+ // Close shuts down the producer; you must call this function before a producer
+ // object passes out of scope, as it may otherwise leak memory.
+ // You must call this before calling Close on the underlying client.
+ Close() error
+
+ // TxnStatus return current producer transaction status.
+ TxnStatus() ProducerTxnStatusFlag
+
+ // IsTransactional return true when current producer is transactional.
+ IsTransactional() bool
+
+ // BeginTxn mark current transaction as ready.
+ BeginTxn() error
+
+ // CommitTxn commit current transaction.
+ CommitTxn() error
+
+ // AbortTxn abort current transaction.
+ AbortTxn() error
+
+ // AddOffsetsToTxn add associated offsets to current transaction.
+ AddOffsetsToTxn(offsets map[string][]*PartitionOffsetMetadata, groupId string) error
+
+ // AddMessageToTxn add message offsets to current transaction.
+ AddMessageToTxn(msg *ConsumerMessage, groupId string, metadata *string) error
+}
+
+type syncProducer struct {
+ producer *asyncProducer
+ wg sync.WaitGroup
+}
+
+// NewSyncProducer creates a new SyncProducer using the given broker addresses and configuration.
+func NewSyncProducer(addrs []string, config *Config) (SyncProducer, error) {
+ if config == nil {
+ config = NewConfig()
+ config.Producer.Return.Successes = true
+ }
+
+ if err := verifyProducerConfig(config); err != nil {
+ return nil, err
+ }
+
+ p, err := NewAsyncProducer(addrs, config)
+ if err != nil {
+ return nil, err
+ }
+ return newSyncProducerFromAsyncProducer(p.(*asyncProducer)), nil
+}
+
+// NewSyncProducerFromClient creates a new SyncProducer using the given client. It is still
+// necessary to call Close() on the underlying client when shutting down this producer.
+func NewSyncProducerFromClient(client Client) (SyncProducer, error) {
+ if err := verifyProducerConfig(client.Config()); err != nil {
+ return nil, err
+ }
+
+ p, err := NewAsyncProducerFromClient(client)
+ if err != nil {
+ return nil, err
+ }
+ return newSyncProducerFromAsyncProducer(p.(*asyncProducer)), nil
+}
+
+func newSyncProducerFromAsyncProducer(p *asyncProducer) *syncProducer {
+ sp := &syncProducer{producer: p}
+
+ sp.wg.Add(2)
+ go withRecover(sp.handleSuccesses)
+ go withRecover(sp.handleErrors)
+
+ return sp
+}
+
+func verifyProducerConfig(config *Config) error {
+ if !config.Producer.Return.Errors {
+ return ConfigurationError("Producer.Return.Errors must be true to be used in a SyncProducer")
+ }
+ if !config.Producer.Return.Successes {
+ return ConfigurationError("Producer.Return.Successes must be true to be used in a SyncProducer")
+ }
+ return nil
+}
+
+func (sp *syncProducer) SendMessage(msg *ProducerMessage) (partition int32, offset int64, err error) {
+ expectation := expectationsPool.Get().(chan *ProducerError)
+ msg.expectation = expectation
+ sp.producer.Input() <- msg
+ pErr := <-expectation
+ msg.expectation = nil
+ expectationsPool.Put(expectation)
+ if pErr != nil {
+ return -1, -1, pErr.Err
+ }
+
+ return msg.Partition, msg.Offset, nil
+}
+
+func (sp *syncProducer) SendMessages(msgs []*ProducerMessage) error {
+ indices := make(chan int, len(msgs))
+ go func() {
+ for i, msg := range msgs {
+ expectation := expectationsPool.Get().(chan *ProducerError)
+ msg.expectation = expectation
+ sp.producer.Input() <- msg
+ indices <- i
+ }
+ close(indices)
+ }()
+
+ var errors ProducerErrors
+ for i := range indices {
+ expectation := msgs[i].expectation
+ pErr := <-expectation
+ msgs[i].expectation = nil
+ expectationsPool.Put(expectation)
+ if pErr != nil {
+ errors = append(errors, pErr)
+ }
+ }
+
+ if len(errors) > 0 {
+ return errors
+ }
+ return nil
+}
+
+func (sp *syncProducer) handleSuccesses() {
+ defer sp.wg.Done()
+ for msg := range sp.producer.Successes() {
+ expectation := msg.expectation
+ expectation <- nil
+ }
+}
+
+func (sp *syncProducer) handleErrors() {
+ defer sp.wg.Done()
+ for err := range sp.producer.Errors() {
+ expectation := err.Msg.expectation
+ expectation <- err
+ }
+}
+
+func (sp *syncProducer) Close() error {
+ sp.producer.AsyncClose()
+ sp.wg.Wait()
+ return nil
+}
+
+func (sp *syncProducer) IsTransactional() bool {
+ return sp.producer.IsTransactional()
+}
+
+func (sp *syncProducer) BeginTxn() error {
+ return sp.producer.BeginTxn()
+}
+
+func (sp *syncProducer) CommitTxn() error {
+ return sp.producer.CommitTxn()
+}
+
+func (sp *syncProducer) AbortTxn() error {
+ return sp.producer.AbortTxn()
+}
+
+func (sp *syncProducer) AddOffsetsToTxn(offsets map[string][]*PartitionOffsetMetadata, groupId string) error {
+ return sp.producer.AddOffsetsToTxn(offsets, groupId)
+}
+
+func (sp *syncProducer) AddMessageToTxn(msg *ConsumerMessage, groupId string, metadata *string) error {
+ return sp.producer.AddMessageToTxn(msg, groupId, metadata)
+}
+
+func (p *syncProducer) TxnStatus() ProducerTxnStatusFlag {
+ return p.producer.TxnStatus()
+}
diff --git a/vendor/github.com/IBM/sarama/timestamp.go b/vendor/github.com/IBM/sarama/timestamp.go
new file mode 100644
index 0000000..372278d
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/timestamp.go
@@ -0,0 +1,40 @@
+package sarama
+
+import (
+ "fmt"
+ "time"
+)
+
+type Timestamp struct {
+ *time.Time
+}
+
+func (t Timestamp) encode(pe packetEncoder) error {
+ timestamp := int64(-1)
+
+ if !t.Before(time.Unix(0, 0)) {
+ timestamp = t.UnixNano() / int64(time.Millisecond)
+ } else if !t.IsZero() {
+ return PacketEncodingError{fmt.Sprintf("invalid timestamp (%v)", t)}
+ }
+
+ pe.putInt64(timestamp)
+ return nil
+}
+
+func (t Timestamp) decode(pd packetDecoder) error {
+ millis, err := pd.getInt64()
+ if err != nil {
+ return err
+ }
+
+ // negative timestamps are invalid, in these cases we should return
+ // a zero time
+ timestamp := time.Time{}
+ if millis >= 0 {
+ timestamp = time.Unix(millis/1000, (millis%1000)*int64(time.Millisecond))
+ }
+
+ *t.Time = timestamp
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/transaction_manager.go b/vendor/github.com/IBM/sarama/transaction_manager.go
new file mode 100644
index 0000000..bf20b75
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/transaction_manager.go
@@ -0,0 +1,930 @@
+package sarama
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+ "sync"
+ "time"
+)
+
+// ProducerTxnStatusFlag mark current transaction status.
+type ProducerTxnStatusFlag int16
+
+const (
+ // ProducerTxnFlagUninitialized when txnmgr is created
+ ProducerTxnFlagUninitialized ProducerTxnStatusFlag = 1 << iota
+ // ProducerTxnFlagInitializing when txnmgr is initializing
+ ProducerTxnFlagInitializing
+ // ProducerTxnFlagReady when is ready to receive transaction
+ ProducerTxnFlagReady
+ // ProducerTxnFlagInTransaction when transaction is started
+ ProducerTxnFlagInTransaction
+ // ProducerTxnFlagEndTransaction when transaction will be committed
+ ProducerTxnFlagEndTransaction
+ // ProducerTxnFlagInError when having abortable or fatal error
+ ProducerTxnFlagInError
+ // ProducerTxnFlagCommittingTransaction when committing txn
+ ProducerTxnFlagCommittingTransaction
+ // ProducerTxnFlagAbortingTransaction when committing txn
+ ProducerTxnFlagAbortingTransaction
+ // ProducerTxnFlagAbortableError when producer encounter an abortable error
+ // Must call AbortTxn in this case.
+ ProducerTxnFlagAbortableError
+ // ProducerTxnFlagFatalError when producer encounter an fatal error
+ // Must Close an recreate it.
+ ProducerTxnFlagFatalError
+)
+
+func (s ProducerTxnStatusFlag) String() string {
+ status := make([]string, 0)
+ if s&ProducerTxnFlagUninitialized != 0 {
+ status = append(status, "ProducerTxnStateUninitialized")
+ }
+ if s&ProducerTxnFlagInitializing != 0 {
+ status = append(status, "ProducerTxnStateInitializing")
+ }
+ if s&ProducerTxnFlagReady != 0 {
+ status = append(status, "ProducerTxnStateReady")
+ }
+ if s&ProducerTxnFlagInTransaction != 0 {
+ status = append(status, "ProducerTxnStateInTransaction")
+ }
+ if s&ProducerTxnFlagEndTransaction != 0 {
+ status = append(status, "ProducerTxnStateEndTransaction")
+ }
+ if s&ProducerTxnFlagInError != 0 {
+ status = append(status, "ProducerTxnStateInError")
+ }
+ if s&ProducerTxnFlagCommittingTransaction != 0 {
+ status = append(status, "ProducerTxnStateCommittingTransaction")
+ }
+ if s&ProducerTxnFlagAbortingTransaction != 0 {
+ status = append(status, "ProducerTxnStateAbortingTransaction")
+ }
+ if s&ProducerTxnFlagAbortableError != 0 {
+ status = append(status, "ProducerTxnStateAbortableError")
+ }
+ if s&ProducerTxnFlagFatalError != 0 {
+ status = append(status, "ProducerTxnStateFatalError")
+ }
+ return strings.Join(status, "|")
+}
+
+// transactionManager keeps the state necessary to ensure idempotent production
+type transactionManager struct {
+ producerID int64
+ producerEpoch int16
+ sequenceNumbers map[string]int32
+ mutex sync.Mutex
+ transactionalID string
+ transactionTimeout time.Duration
+ client Client
+
+ // when kafka cluster is at least 2.5.0.
+ // used to recover when producer failed.
+ coordinatorSupportsBumpingEpoch bool
+
+ // When producer need to bump it's epoch.
+ epochBumpRequired bool
+ // Record last seen error.
+ lastError error
+
+ // Ensure that status is never accessed with a race-condition.
+ statusLock sync.RWMutex
+ status ProducerTxnStatusFlag
+
+ // Ensure that only one goroutine will update partitions in current transaction.
+ partitionInTxnLock sync.Mutex
+ pendingPartitionsInCurrentTxn topicPartitionSet
+ partitionsInCurrentTxn topicPartitionSet
+
+ // Offsets to add to transaction.
+ offsetsInCurrentTxn map[string]topicPartitionOffsets
+}
+
+const (
+ noProducerID = -1
+ noProducerEpoch = -1
+
+ // see publishTxnPartitions comment.
+ addPartitionsRetryBackoff = 20 * time.Millisecond
+)
+
+// txnmngr allowed transitions.
+var producerTxnTransitions = map[ProducerTxnStatusFlag][]ProducerTxnStatusFlag{
+ ProducerTxnFlagUninitialized: {
+ ProducerTxnFlagReady,
+ ProducerTxnFlagInError,
+ },
+ // When we need are initializing
+ ProducerTxnFlagInitializing: {
+ ProducerTxnFlagInitializing,
+ ProducerTxnFlagReady,
+ ProducerTxnFlagInError,
+ },
+ // When we have initialized transactional producer
+ ProducerTxnFlagReady: {
+ ProducerTxnFlagInTransaction,
+ },
+ // When beginTxn has been called
+ ProducerTxnFlagInTransaction: {
+ // When calling commit or abort
+ ProducerTxnFlagEndTransaction,
+ // When got an error
+ ProducerTxnFlagInError,
+ },
+ ProducerTxnFlagEndTransaction: {
+ // When epoch bump
+ ProducerTxnFlagInitializing,
+ // When commit is good
+ ProducerTxnFlagReady,
+ // When got an error
+ ProducerTxnFlagInError,
+ },
+ // Need to abort transaction
+ ProducerTxnFlagAbortableError: {
+ // Call AbortTxn
+ ProducerTxnFlagAbortingTransaction,
+ // When got an error
+ ProducerTxnFlagInError,
+ },
+ // Need to close producer
+ ProducerTxnFlagFatalError: {
+ ProducerTxnFlagFatalError,
+ },
+}
+
+type topicPartition struct {
+ topic string
+ partition int32
+}
+
+// to ensure that we don't do a full scan every time a partition or an offset is added.
+type (
+ topicPartitionSet map[topicPartition]struct{}
+ topicPartitionOffsets map[topicPartition]*PartitionOffsetMetadata
+)
+
+func (s topicPartitionSet) mapToRequest() map[string][]int32 {
+ result := make(map[string][]int32, len(s))
+ for tp := range s {
+ result[tp.topic] = append(result[tp.topic], tp.partition)
+ }
+ return result
+}
+
+func (s topicPartitionOffsets) mapToRequest() map[string][]*PartitionOffsetMetadata {
+ result := make(map[string][]*PartitionOffsetMetadata, len(s))
+ for tp, offset := range s {
+ result[tp.topic] = append(result[tp.topic], offset)
+ }
+ return result
+}
+
+// Return true if current transition is allowed.
+func (t *transactionManager) isTransitionValid(target ProducerTxnStatusFlag) bool {
+ for status, allowedTransitions := range producerTxnTransitions {
+ if status&t.status != 0 {
+ for _, allowedTransition := range allowedTransitions {
+ if allowedTransition&target != 0 {
+ return true
+ }
+ }
+ }
+ }
+ return false
+}
+
+// Get current transaction status.
+func (t *transactionManager) currentTxnStatus() ProducerTxnStatusFlag {
+ t.statusLock.RLock()
+ defer t.statusLock.RUnlock()
+
+ return t.status
+}
+
+// Try to transition to a valid status and return an error otherwise.
+func (t *transactionManager) transitionTo(target ProducerTxnStatusFlag, err error) error {
+ t.statusLock.Lock()
+ defer t.statusLock.Unlock()
+
+ if !t.isTransitionValid(target) {
+ return ErrTransitionNotAllowed
+ }
+
+ if target&ProducerTxnFlagInError != 0 {
+ if err == nil {
+ return ErrCannotTransitionNilError
+ }
+ t.lastError = err
+ } else {
+ t.lastError = nil
+ }
+
+ DebugLogger.Printf("txnmgr/transition [%s] transition from %s to %s\n", t.transactionalID, t.status, target)
+
+ t.status = target
+ return err
+}
+
+func (t *transactionManager) getAndIncrementSequenceNumber(topic string, partition int32) (int32, int16) {
+ key := fmt.Sprintf("%s-%d", topic, partition)
+ t.mutex.Lock()
+ defer t.mutex.Unlock()
+ sequence := t.sequenceNumbers[key]
+ t.sequenceNumbers[key] = sequence + 1
+ return sequence, t.producerEpoch
+}
+
+func (t *transactionManager) bumpEpoch() {
+ t.mutex.Lock()
+ defer t.mutex.Unlock()
+ t.producerEpoch++
+ for k := range t.sequenceNumbers {
+ t.sequenceNumbers[k] = 0
+ }
+}
+
+func (t *transactionManager) getProducerID() (int64, int16) {
+ t.mutex.Lock()
+ defer t.mutex.Unlock()
+ return t.producerID, t.producerEpoch
+}
+
+// Compute retry backoff considered current attempts.
+func (t *transactionManager) computeBackoff(attemptsRemaining int) time.Duration {
+ if t.client.Config().Producer.Transaction.Retry.BackoffFunc != nil {
+ maxRetries := t.client.Config().Producer.Transaction.Retry.Max
+ retries := maxRetries - attemptsRemaining
+ return t.client.Config().Producer.Transaction.Retry.BackoffFunc(retries, maxRetries)
+ }
+ return t.client.Config().Producer.Transaction.Retry.Backoff
+}
+
+// return true is txnmngr is transactinal.
+func (t *transactionManager) isTransactional() bool {
+ return t.transactionalID != ""
+}
+
+// add specified offsets to current transaction.
+func (t *transactionManager) addOffsetsToTxn(offsetsToAdd map[string][]*PartitionOffsetMetadata, groupId string) error {
+ t.mutex.Lock()
+ defer t.mutex.Unlock()
+
+ if t.currentTxnStatus()&ProducerTxnFlagInTransaction == 0 {
+ return ErrTransactionNotReady
+ }
+
+ if t.currentTxnStatus()&ProducerTxnFlagFatalError != 0 {
+ return t.lastError
+ }
+
+ if _, ok := t.offsetsInCurrentTxn[groupId]; !ok {
+ t.offsetsInCurrentTxn[groupId] = topicPartitionOffsets{}
+ }
+
+ for topic, offsets := range offsetsToAdd {
+ for _, offset := range offsets {
+ tp := topicPartition{topic: topic, partition: offset.Partition}
+ t.offsetsInCurrentTxn[groupId][tp] = offset
+ }
+ }
+ return nil
+}
+
+// send txnmgnr save offsets to transaction coordinator.
+func (t *transactionManager) publishOffsetsToTxn(offsets topicPartitionOffsets, groupId string) (topicPartitionOffsets, error) {
+ // First AddOffsetsToTxn
+ attemptsRemaining := t.client.Config().Producer.Transaction.Retry.Max
+ exec := func(run func() (bool, error), err error) error {
+ for attemptsRemaining >= 0 {
+ var retry bool
+ retry, err = run()
+ if !retry {
+ return err
+ }
+ backoff := t.computeBackoff(attemptsRemaining)
+ Logger.Printf("txnmgr/add-offset-to-txn [%s] retrying after %dms... (%d attempts remaining) (%s)\n",
+ t.transactionalID, backoff/time.Millisecond, attemptsRemaining, err)
+ time.Sleep(backoff)
+ attemptsRemaining--
+ }
+ return err
+ }
+ lastError := exec(func() (bool, error) {
+ coordinator, err := t.client.TransactionCoordinator(t.transactionalID)
+ if err != nil {
+ return true, err
+ }
+ request := &AddOffsetsToTxnRequest{
+ TransactionalID: t.transactionalID,
+ ProducerEpoch: t.producerEpoch,
+ ProducerID: t.producerID,
+ GroupID: groupId,
+ }
+ if t.client.Config().Version.IsAtLeast(V2_7_0_0) {
+ // Version 2 adds the support for new error code PRODUCER_FENCED.
+ request.Version = 2
+ } else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {
+ // Version 1 is the same as version 0.
+ request.Version = 1
+ }
+ response, err := coordinator.AddOffsetsToTxn(request)
+ if err != nil {
+ // If an error occurred try to refresh current transaction coordinator.
+ _ = coordinator.Close()
+ _ = t.client.RefreshTransactionCoordinator(t.transactionalID)
+ return true, err
+ }
+ if response == nil {
+ // If no response is returned just retry.
+ return true, ErrTxnUnableToParseResponse
+ }
+ if response.Err == ErrNoError {
+ DebugLogger.Printf("txnmgr/add-offset-to-txn [%s] successful add-offset-to-txn with group %s %+v\n",
+ t.transactionalID, groupId, response)
+ // If no error, just exit.
+ return false, nil
+ }
+ switch response.Err {
+ case ErrConsumerCoordinatorNotAvailable:
+ fallthrough
+ case ErrNotCoordinatorForConsumer:
+ _ = coordinator.Close()
+ _ = t.client.RefreshTransactionCoordinator(t.transactionalID)
+ fallthrough
+ case ErrOffsetsLoadInProgress:
+ fallthrough
+ case ErrConcurrentTransactions:
+ // Retry
+ case ErrUnknownProducerID:
+ fallthrough
+ case ErrInvalidProducerIDMapping:
+ return false, t.abortableErrorIfPossible(response.Err)
+ case ErrGroupAuthorizationFailed:
+ return false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagAbortableError, response.Err)
+ default:
+ // Others are fatal
+ return false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, response.Err)
+ }
+ return true, response.Err
+ }, nil)
+
+ if lastError != nil {
+ return offsets, lastError
+ }
+
+ resultOffsets := offsets
+ // Then TxnOffsetCommit
+ // note the result is not completed until the TxnOffsetCommit returns
+ attemptsRemaining = t.client.Config().Producer.Transaction.Retry.Max
+ execTxnOffsetCommit := func(run func() (topicPartitionOffsets, bool, error), err error) (topicPartitionOffsets, error) {
+ var r topicPartitionOffsets
+ for attemptsRemaining >= 0 {
+ var retry bool
+ r, retry, err = run()
+ if !retry {
+ return r, err
+ }
+ backoff := t.computeBackoff(attemptsRemaining)
+ Logger.Printf("txnmgr/txn-offset-commit [%s] retrying after %dms... (%d attempts remaining) (%s)\n",
+ t.transactionalID, backoff/time.Millisecond, attemptsRemaining, err)
+ time.Sleep(backoff)
+ attemptsRemaining--
+ }
+ return r, err
+ }
+ return execTxnOffsetCommit(func() (topicPartitionOffsets, bool, error) {
+ consumerGroupCoordinator, err := t.client.Coordinator(groupId)
+ if err != nil {
+ return resultOffsets, true, err
+ }
+ request := &TxnOffsetCommitRequest{
+ TransactionalID: t.transactionalID,
+ ProducerEpoch: t.producerEpoch,
+ ProducerID: t.producerID,
+ GroupID: groupId,
+ Topics: offsets.mapToRequest(),
+ }
+ if t.client.Config().Version.IsAtLeast(V2_1_0_0) {
+ // Version 2 adds the committed leader epoch.
+ request.Version = 2
+ } else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {
+ // Version 1 is the same as version 0.
+ request.Version = 1
+ }
+ responses, err := consumerGroupCoordinator.TxnOffsetCommit(request)
+ if err != nil {
+ _ = consumerGroupCoordinator.Close()
+ _ = t.client.RefreshCoordinator(groupId)
+ return resultOffsets, true, err
+ }
+
+ if responses == nil {
+ return resultOffsets, true, ErrTxnUnableToParseResponse
+ }
+
+ var responseErrors []error
+ failedTxn := topicPartitionOffsets{}
+ for topic, partitionErrors := range responses.Topics {
+ for _, partitionError := range partitionErrors {
+ switch partitionError.Err {
+ case ErrNoError:
+ continue
+ // If the topic is unknown or the coordinator is loading, retry with the current coordinator
+ case ErrRequestTimedOut:
+ fallthrough
+ case ErrConsumerCoordinatorNotAvailable:
+ fallthrough
+ case ErrNotCoordinatorForConsumer:
+ _ = consumerGroupCoordinator.Close()
+ _ = t.client.RefreshCoordinator(groupId)
+ fallthrough
+ case ErrUnknownTopicOrPartition:
+ fallthrough
+ case ErrOffsetsLoadInProgress:
+ // Do nothing just retry
+ case ErrIllegalGeneration:
+ fallthrough
+ case ErrUnknownMemberId:
+ fallthrough
+ case ErrFencedInstancedId:
+ fallthrough
+ case ErrGroupAuthorizationFailed:
+ return resultOffsets, false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagAbortableError, partitionError.Err)
+ default:
+ // Others are fatal
+ return resultOffsets, false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, partitionError.Err)
+ }
+ tp := topicPartition{topic: topic, partition: partitionError.Partition}
+ failedTxn[tp] = offsets[tp]
+ responseErrors = append(responseErrors, partitionError.Err)
+ }
+ }
+
+ resultOffsets = failedTxn
+
+ if len(resultOffsets) == 0 {
+ DebugLogger.Printf("txnmgr/txn-offset-commit [%s] successful txn-offset-commit with group %s\n",
+ t.transactionalID, groupId)
+ return resultOffsets, false, nil
+ }
+ return resultOffsets, true, Wrap(ErrTxnOffsetCommit, responseErrors...)
+ }, nil)
+}
+
+func (t *transactionManager) initProducerId() (int64, int16, error) {
+ isEpochBump := false
+
+ req := &InitProducerIDRequest{}
+ if t.isTransactional() {
+ req.TransactionalID = &t.transactionalID
+ req.TransactionTimeout = t.transactionTimeout
+ }
+
+ if t.client.Config().Version.IsAtLeast(V2_5_0_0) {
+ if t.client.Config().Version.IsAtLeast(V2_7_0_0) {
+ // Version 4 adds the support for new error code PRODUCER_FENCED.
+ req.Version = 4
+ } else {
+ // Version 3 adds ProducerId and ProducerEpoch, allowing producers to try
+ // to resume after an INVALID_PRODUCER_EPOCH error
+ req.Version = 3
+ }
+ isEpochBump = t.producerID != noProducerID && t.producerEpoch != noProducerEpoch
+ t.coordinatorSupportsBumpingEpoch = true
+ req.ProducerID = t.producerID
+ req.ProducerEpoch = t.producerEpoch
+ } else if t.client.Config().Version.IsAtLeast(V2_4_0_0) {
+ // Version 2 is the first flexible version.
+ req.Version = 2
+ } else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {
+ // Version 1 is the same as version 0.
+ req.Version = 1
+ }
+
+ if isEpochBump {
+ err := t.transitionTo(ProducerTxnFlagInitializing, nil)
+ if err != nil {
+ return -1, -1, err
+ }
+ DebugLogger.Printf("txnmgr/init-producer-id [%s] invoking InitProducerId for the first time in order to acquire a producer ID\n",
+ t.transactionalID)
+ } else {
+ DebugLogger.Printf("txnmgr/init-producer-id [%s] invoking InitProducerId with current producer ID %d and epoch %d in order to bump the epoch\n",
+ t.transactionalID, t.producerID, t.producerEpoch)
+ }
+
+ attemptsRemaining := t.client.Config().Producer.Transaction.Retry.Max
+ exec := func(run func() (int64, int16, bool, error), err error) (int64, int16, error) {
+ pid := int64(-1)
+ pepoch := int16(-1)
+ for attemptsRemaining >= 0 {
+ var retry bool
+ pid, pepoch, retry, err = run()
+ if !retry {
+ return pid, pepoch, err
+ }
+ backoff := t.computeBackoff(attemptsRemaining)
+ Logger.Printf("txnmgr/init-producer-id [%s] retrying after %dms... (%d attempts remaining) (%s)\n",
+ t.transactionalID, backoff/time.Millisecond, attemptsRemaining, err)
+ time.Sleep(backoff)
+ attemptsRemaining--
+ }
+ return -1, -1, err
+ }
+ return exec(func() (int64, int16, bool, error) {
+ var err error
+ var coordinator *Broker
+ if t.isTransactional() {
+ coordinator, err = t.client.TransactionCoordinator(t.transactionalID)
+ } else {
+ coordinator = t.client.LeastLoadedBroker()
+ }
+ if err != nil {
+ return -1, -1, true, err
+ }
+ response, err := coordinator.InitProducerID(req)
+ if err != nil {
+ if t.isTransactional() {
+ _ = coordinator.Close()
+ _ = t.client.RefreshTransactionCoordinator(t.transactionalID)
+ }
+ return -1, -1, true, err
+ }
+ if response == nil {
+ return -1, -1, true, ErrTxnUnableToParseResponse
+ }
+ if response.Err == ErrNoError {
+ if isEpochBump {
+ t.sequenceNumbers = make(map[string]int32)
+ }
+ err := t.transitionTo(ProducerTxnFlagReady, nil)
+ if err != nil {
+ return -1, -1, true, err
+ }
+ DebugLogger.Printf("txnmgr/init-producer-id [%s] successful init producer id %+v\n",
+ t.transactionalID, response)
+ return response.ProducerID, response.ProducerEpoch, false, nil
+ }
+ switch response.Err {
+ // Retriable errors
+ case ErrConsumerCoordinatorNotAvailable, ErrNotCoordinatorForConsumer, ErrOffsetsLoadInProgress:
+ if t.isTransactional() {
+ _ = coordinator.Close()
+ _ = t.client.RefreshTransactionCoordinator(t.transactionalID)
+ }
+ // Fatal errors
+ default:
+ return -1, -1, false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, response.Err)
+ }
+ return -1, -1, true, response.Err
+ }, nil)
+}
+
+// if kafka cluster is at least 2.5.0 mark txnmngr to bump epoch else mark it as fatal.
+func (t *transactionManager) abortableErrorIfPossible(err error) error {
+ if t.coordinatorSupportsBumpingEpoch {
+ t.epochBumpRequired = true
+ return t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagAbortableError, err)
+ }
+ return t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, err)
+}
+
+// End current transaction.
+func (t *transactionManager) completeTransaction() error {
+ if t.epochBumpRequired {
+ err := t.transitionTo(ProducerTxnFlagInitializing, nil)
+ if err != nil {
+ return err
+ }
+ } else {
+ err := t.transitionTo(ProducerTxnFlagReady, nil)
+ if err != nil {
+ return err
+ }
+ }
+
+ t.lastError = nil
+ t.epochBumpRequired = false
+ t.partitionsInCurrentTxn = topicPartitionSet{}
+ t.pendingPartitionsInCurrentTxn = topicPartitionSet{}
+ t.offsetsInCurrentTxn = map[string]topicPartitionOffsets{}
+
+ return nil
+}
+
+// send EndTxn request with commit flag. (true when committing false otherwise)
+func (t *transactionManager) endTxn(commit bool) error {
+ attemptsRemaining := t.client.Config().Producer.Transaction.Retry.Max
+ exec := func(run func() (bool, error), err error) error {
+ for attemptsRemaining >= 0 {
+ var retry bool
+ retry, err = run()
+ if !retry {
+ return err
+ }
+ backoff := t.computeBackoff(attemptsRemaining)
+ Logger.Printf("txnmgr/endtxn [%s] retrying after %dms... (%d attempts remaining) (%s)\n",
+ t.transactionalID, backoff/time.Millisecond, attemptsRemaining, err)
+ time.Sleep(backoff)
+ attemptsRemaining--
+ }
+ return err
+ }
+ return exec(func() (bool, error) {
+ coordinator, err := t.client.TransactionCoordinator(t.transactionalID)
+ if err != nil {
+ return true, err
+ }
+ request := &EndTxnRequest{
+ TransactionalID: t.transactionalID,
+ ProducerEpoch: t.producerEpoch,
+ ProducerID: t.producerID,
+ TransactionResult: commit,
+ }
+ if t.client.Config().Version.IsAtLeast(V2_7_0_0) {
+ // Version 2 adds the support for new error code PRODUCER_FENCED.
+ request.Version = 2
+ } else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {
+ // Version 1 is the same as version 0.
+ request.Version = 1
+ }
+ response, err := coordinator.EndTxn(request)
+ if err != nil {
+ // Always retry on network error
+ _ = coordinator.Close()
+ _ = t.client.RefreshTransactionCoordinator(t.transactionalID)
+ return true, err
+ }
+ if response == nil {
+ return true, ErrTxnUnableToParseResponse
+ }
+ if response.Err == ErrNoError {
+ DebugLogger.Printf("txnmgr/endtxn [%s] successful to end txn %+v\n",
+ t.transactionalID, response)
+ return false, t.completeTransaction()
+ }
+ switch response.Err {
+ // Need to refresh coordinator
+ case ErrConsumerCoordinatorNotAvailable:
+ fallthrough
+ case ErrNotCoordinatorForConsumer:
+ _ = coordinator.Close()
+ _ = t.client.RefreshTransactionCoordinator(t.transactionalID)
+ fallthrough
+ case ErrOffsetsLoadInProgress:
+ fallthrough
+ case ErrConcurrentTransactions:
+ // Just retry
+ case ErrUnknownProducerID:
+ fallthrough
+ case ErrInvalidProducerIDMapping:
+ return false, t.abortableErrorIfPossible(response.Err)
+ // Fatal errors
+ default:
+ return false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, response.Err)
+ }
+ return true, response.Err
+ }, nil)
+}
+
+// We will try to publish associated offsets for each groups
+// then send endtxn request to mark transaction as finished.
+func (t *transactionManager) finishTransaction(commit bool) error {
+ t.mutex.Lock()
+ defer t.mutex.Unlock()
+
+ // Ensure no error when committing or aborting
+ if commit && t.currentTxnStatus()&ProducerTxnFlagInError != 0 {
+ return t.lastError
+ } else if !commit && t.currentTxnStatus()&ProducerTxnFlagFatalError != 0 {
+ return t.lastError
+ }
+
+ // if no records has been sent don't do anything.
+ if len(t.partitionsInCurrentTxn) == 0 {
+ return t.completeTransaction()
+ }
+
+ epochBump := t.epochBumpRequired
+ // If we're aborting the transaction, so there should be no need to add offsets.
+ if commit && len(t.offsetsInCurrentTxn) > 0 {
+ for group, offsets := range t.offsetsInCurrentTxn {
+ newOffsets, err := t.publishOffsetsToTxn(offsets, group)
+ if err != nil {
+ t.offsetsInCurrentTxn[group] = newOffsets
+ return err
+ }
+ delete(t.offsetsInCurrentTxn, group)
+ }
+ }
+
+ if t.currentTxnStatus()&ProducerTxnFlagFatalError != 0 {
+ return t.lastError
+ }
+
+ if !errors.Is(t.lastError, ErrInvalidProducerIDMapping) {
+ err := t.endTxn(commit)
+ if err != nil {
+ return err
+ }
+ if !epochBump {
+ return nil
+ }
+ }
+ // reset pid and epoch if needed.
+ return t.initializeTransactions()
+}
+
+// called before sending any transactional record
+// won't do anything if current topic-partition is already added to transaction.
+func (t *transactionManager) maybeAddPartitionToCurrentTxn(topic string, partition int32) {
+ if t.currentTxnStatus()&ProducerTxnFlagInError != 0 {
+ return
+ }
+
+ tp := topicPartition{topic: topic, partition: partition}
+
+ t.partitionInTxnLock.Lock()
+ defer t.partitionInTxnLock.Unlock()
+ if _, ok := t.partitionsInCurrentTxn[tp]; ok {
+ // partition is already added
+ return
+ }
+
+ t.pendingPartitionsInCurrentTxn[tp] = struct{}{}
+}
+
+// Makes a request to kafka to add a list of partitions ot the current transaction.
+func (t *transactionManager) publishTxnPartitions() error {
+ t.partitionInTxnLock.Lock()
+ defer t.partitionInTxnLock.Unlock()
+
+ if t.currentTxnStatus()&ProducerTxnFlagInError != 0 {
+ return t.lastError
+ }
+
+ if len(t.pendingPartitionsInCurrentTxn) == 0 {
+ return nil
+ }
+
+ // Remove the partitions from the pending set regardless of the result. We use the presence
+ // of partitions in the pending set to know when it is not safe to send batches. However, if
+ // the partitions failed to be added and we enter an error state, we expect the batches to be
+ // aborted anyway. In this case, we must be able to continue sending the batches which are in
+ // retry for partitions that were successfully added.
+ removeAllPartitionsOnFatalOrAbortedError := func() {
+ t.pendingPartitionsInCurrentTxn = topicPartitionSet{}
+ }
+
+ // We only want to reduce the backoff when retrying the first AddPartition which errored out due to a
+ // CONCURRENT_TRANSACTIONS error since this means that the previous transaction is still completing and
+ // we don't want to wait too long before trying to start the new one.
+ //
+ // This is only a temporary fix, the long term solution is being tracked in
+ // https://issues.apache.org/jira/browse/KAFKA-5482
+ retryBackoff := t.client.Config().Producer.Transaction.Retry.Backoff
+ computeBackoff := func(attemptsRemaining int) time.Duration {
+ if t.client.Config().Producer.Transaction.Retry.BackoffFunc != nil {
+ maxRetries := t.client.Config().Producer.Transaction.Retry.Max
+ retries := maxRetries - attemptsRemaining
+ return t.client.Config().Producer.Transaction.Retry.BackoffFunc(retries, maxRetries)
+ }
+ return retryBackoff
+ }
+ attemptsRemaining := t.client.Config().Producer.Transaction.Retry.Max
+
+ exec := func(run func() (bool, error), err error) error {
+ for attemptsRemaining >= 0 {
+ var retry bool
+ retry, err = run()
+ if !retry {
+ return err
+ }
+ backoff := computeBackoff(attemptsRemaining)
+ Logger.Printf("txnmgr/add-partition-to-txn retrying after %dms... (%d attempts remaining) (%s)\n", backoff/time.Millisecond, attemptsRemaining, err)
+ time.Sleep(backoff)
+ attemptsRemaining--
+ }
+ return err
+ }
+ return exec(func() (bool, error) {
+ coordinator, err := t.client.TransactionCoordinator(t.transactionalID)
+ if err != nil {
+ return true, err
+ }
+ request := &AddPartitionsToTxnRequest{
+ TransactionalID: t.transactionalID,
+ ProducerID: t.producerID,
+ ProducerEpoch: t.producerEpoch,
+ TopicPartitions: t.pendingPartitionsInCurrentTxn.mapToRequest(),
+ }
+ if t.client.Config().Version.IsAtLeast(V2_7_0_0) {
+ // Version 2 adds the support for new error code PRODUCER_FENCED.
+ request.Version = 2
+ } else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {
+ // Version 1 is the same as version 0.
+ request.Version = 1
+ }
+ addPartResponse, err := coordinator.AddPartitionsToTxn(request)
+ if err != nil {
+ _ = coordinator.Close()
+ _ = t.client.RefreshTransactionCoordinator(t.transactionalID)
+ return true, err
+ }
+
+ if addPartResponse == nil {
+ return true, ErrTxnUnableToParseResponse
+ }
+
+ // remove from the list partitions that have been successfully updated
+ var responseErrors []error
+ for topic, results := range addPartResponse.Errors {
+ for _, response := range results {
+ tp := topicPartition{topic: topic, partition: response.Partition}
+ switch response.Err {
+ case ErrNoError:
+ // Mark partition as added to transaction
+ t.partitionsInCurrentTxn[tp] = struct{}{}
+ delete(t.pendingPartitionsInCurrentTxn, tp)
+ continue
+ case ErrConsumerCoordinatorNotAvailable:
+ fallthrough
+ case ErrNotCoordinatorForConsumer:
+ _ = coordinator.Close()
+ _ = t.client.RefreshTransactionCoordinator(t.transactionalID)
+ fallthrough
+ case ErrUnknownTopicOrPartition:
+ fallthrough
+ case ErrOffsetsLoadInProgress:
+ // Retry topicPartition
+ case ErrConcurrentTransactions:
+ if len(t.partitionsInCurrentTxn) == 0 && retryBackoff > addPartitionsRetryBackoff {
+ retryBackoff = addPartitionsRetryBackoff
+ }
+ case ErrOperationNotAttempted:
+ fallthrough
+ case ErrTopicAuthorizationFailed:
+ removeAllPartitionsOnFatalOrAbortedError()
+ return false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagAbortableError, response.Err)
+ case ErrUnknownProducerID:
+ fallthrough
+ case ErrInvalidProducerIDMapping:
+ removeAllPartitionsOnFatalOrAbortedError()
+ return false, t.abortableErrorIfPossible(response.Err)
+ // Fatal errors
+ default:
+ removeAllPartitionsOnFatalOrAbortedError()
+ return false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, response.Err)
+ }
+ responseErrors = append(responseErrors, response.Err)
+ }
+ }
+
+ // handle end
+ if len(t.pendingPartitionsInCurrentTxn) == 0 {
+ DebugLogger.Printf("txnmgr/add-partition-to-txn [%s] successful to add partitions txn %+v\n",
+ t.transactionalID, addPartResponse)
+ return false, nil
+ }
+ return true, Wrap(ErrAddPartitionsToTxn, responseErrors...)
+ }, nil)
+}
+
+// Build a new transaction manager sharing producer client.
+func newTransactionManager(conf *Config, client Client) (*transactionManager, error) {
+ txnmgr := &transactionManager{
+ producerID: noProducerID,
+ producerEpoch: noProducerEpoch,
+ client: client,
+ pendingPartitionsInCurrentTxn: topicPartitionSet{},
+ partitionsInCurrentTxn: topicPartitionSet{},
+ offsetsInCurrentTxn: make(map[string]topicPartitionOffsets),
+ status: ProducerTxnFlagUninitialized,
+ }
+
+ if conf.Producer.Idempotent {
+ txnmgr.transactionalID = conf.Producer.Transaction.ID
+ txnmgr.transactionTimeout = conf.Producer.Transaction.Timeout
+ txnmgr.sequenceNumbers = make(map[string]int32)
+ txnmgr.mutex = sync.Mutex{}
+
+ var err error
+ txnmgr.producerID, txnmgr.producerEpoch, err = txnmgr.initProducerId()
+ if err != nil {
+ return nil, err
+ }
+ Logger.Printf("txnmgr/init-producer-id [%s] obtained a ProducerId: %d and ProducerEpoch: %d\n",
+ txnmgr.transactionalID, txnmgr.producerID, txnmgr.producerEpoch)
+ }
+
+ return txnmgr, nil
+}
+
+// re-init producer-id and producer-epoch if needed.
+func (t *transactionManager) initializeTransactions() (err error) {
+ t.producerID, t.producerEpoch, err = t.initProducerId()
+ return
+}
diff --git a/vendor/github.com/IBM/sarama/txn_offset_commit_request.go b/vendor/github.com/IBM/sarama/txn_offset_commit_request.go
new file mode 100644
index 0000000..f390172
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/txn_offset_commit_request.go
@@ -0,0 +1,166 @@
+package sarama
+
+type TxnOffsetCommitRequest struct {
+ Version int16
+ TransactionalID string
+ GroupID string
+ ProducerID int64
+ ProducerEpoch int16
+ Topics map[string][]*PartitionOffsetMetadata
+}
+
+func (t *TxnOffsetCommitRequest) setVersion(v int16) {
+ t.Version = v
+}
+
+func (t *TxnOffsetCommitRequest) encode(pe packetEncoder) error {
+ if err := pe.putString(t.TransactionalID); err != nil {
+ return err
+ }
+ if err := pe.putString(t.GroupID); err != nil {
+ return err
+ }
+ pe.putInt64(t.ProducerID)
+ pe.putInt16(t.ProducerEpoch)
+
+ if err := pe.putArrayLength(len(t.Topics)); err != nil {
+ return err
+ }
+ for topic, partitions := range t.Topics {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putArrayLength(len(partitions)); err != nil {
+ return err
+ }
+ for _, partition := range partitions {
+ if err := partition.encode(pe, t.Version); err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func (t *TxnOffsetCommitRequest) decode(pd packetDecoder, version int16) (err error) {
+ t.Version = version
+ if t.TransactionalID, err = pd.getString(); err != nil {
+ return err
+ }
+ if t.GroupID, err = pd.getString(); err != nil {
+ return err
+ }
+ if t.ProducerID, err = pd.getInt64(); err != nil {
+ return err
+ }
+ if t.ProducerEpoch, err = pd.getInt16(); err != nil {
+ return err
+ }
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ t.Topics = make(map[string][]*PartitionOffsetMetadata)
+ for i := 0; i < n; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ m, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ t.Topics[topic] = make([]*PartitionOffsetMetadata, m)
+
+ for j := 0; j < m; j++ {
+ partitionOffsetMetadata := new(PartitionOffsetMetadata)
+ if err := partitionOffsetMetadata.decode(pd, version); err != nil {
+ return err
+ }
+ t.Topics[topic][j] = partitionOffsetMetadata
+ }
+ }
+
+ return nil
+}
+
+func (a *TxnOffsetCommitRequest) key() int16 {
+ return apiKeyTxnOffsetCommit
+}
+
+func (a *TxnOffsetCommitRequest) version() int16 {
+ return a.Version
+}
+
+func (a *TxnOffsetCommitRequest) headerVersion() int16 {
+ return 1
+}
+
+func (a *TxnOffsetCommitRequest) isValidVersion() bool {
+ return a.Version >= 0 && a.Version <= 2
+}
+
+func (a *TxnOffsetCommitRequest) requiredVersion() KafkaVersion {
+ switch a.Version {
+ case 2:
+ return V2_1_0_0
+ case 1:
+ return V2_0_0_0
+ case 0:
+ return V0_11_0_0
+ default:
+ return V2_1_0_0
+ }
+}
+
+type PartitionOffsetMetadata struct {
+ // Partition contains the index of the partition within the topic.
+ Partition int32
+ // Offset contains the message offset to be committed.
+ Offset int64
+ // LeaderEpoch contains the leader epoch of the last consumed record.
+ LeaderEpoch int32
+ // Metadata contains any associated metadata the client wants to keep.
+ Metadata *string
+}
+
+func (p *PartitionOffsetMetadata) encode(pe packetEncoder, version int16) error {
+ pe.putInt32(p.Partition)
+ pe.putInt64(p.Offset)
+
+ if version >= 2 {
+ pe.putInt32(p.LeaderEpoch)
+ }
+
+ if err := pe.putNullableString(p.Metadata); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (p *PartitionOffsetMetadata) decode(pd packetDecoder, version int16) (err error) {
+ if p.Partition, err = pd.getInt32(); err != nil {
+ return err
+ }
+ if p.Offset, err = pd.getInt64(); err != nil {
+ return err
+ }
+
+ if version >= 2 {
+ if p.LeaderEpoch, err = pd.getInt32(); err != nil {
+ return err
+ }
+ }
+
+ if p.Metadata, err = pd.getNullableString(); err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/vendor/github.com/IBM/sarama/txn_offset_commit_response.go b/vendor/github.com/IBM/sarama/txn_offset_commit_response.go
new file mode 100644
index 0000000..19bcad3
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/txn_offset_commit_response.go
@@ -0,0 +1,110 @@
+package sarama
+
+import (
+ "time"
+)
+
+type TxnOffsetCommitResponse struct {
+ Version int16
+ ThrottleTime time.Duration
+ Topics map[string][]*PartitionError
+}
+
+func (t *TxnOffsetCommitResponse) setVersion(v int16) {
+ t.Version = v
+}
+
+func (t *TxnOffsetCommitResponse) encode(pe packetEncoder) error {
+ pe.putDurationMs(t.ThrottleTime)
+ if err := pe.putArrayLength(len(t.Topics)); err != nil {
+ return err
+ }
+
+ for topic, e := range t.Topics {
+ if err := pe.putString(topic); err != nil {
+ return err
+ }
+ if err := pe.putArrayLength(len(e)); err != nil {
+ return err
+ }
+ for _, partitionError := range e {
+ if err := partitionError.encode(pe); err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func (t *TxnOffsetCommitResponse) decode(pd packetDecoder, version int16) (err error) {
+ t.Version = version
+ throttleTime, err := pd.getInt32()
+ if err != nil {
+ return err
+ }
+ t.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
+
+ n, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ t.Topics = make(map[string][]*PartitionError)
+
+ for i := 0; i < n; i++ {
+ topic, err := pd.getString()
+ if err != nil {
+ return err
+ }
+
+ m, err := pd.getArrayLength()
+ if err != nil {
+ return err
+ }
+
+ t.Topics[topic] = make([]*PartitionError, m)
+
+ for j := 0; j < m; j++ {
+ t.Topics[topic][j] = new(PartitionError)
+ if err := t.Topics[topic][j].decode(pd, version); err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func (a *TxnOffsetCommitResponse) key() int16 {
+ return apiKeyTxnOffsetCommit
+}
+
+func (a *TxnOffsetCommitResponse) version() int16 {
+ return a.Version
+}
+
+func (a *TxnOffsetCommitResponse) headerVersion() int16 {
+ return 0
+}
+
+func (a *TxnOffsetCommitResponse) isValidVersion() bool {
+ return a.Version >= 0 && a.Version <= 2
+}
+
+func (a *TxnOffsetCommitResponse) requiredVersion() KafkaVersion {
+ switch a.Version {
+ case 2:
+ return V2_1_0_0
+ case 1:
+ return V2_0_0_0
+ case 0:
+ return V0_11_0_0
+ default:
+ return V2_1_0_0
+ }
+}
+
+func (r *TxnOffsetCommitResponse) throttleTime() time.Duration {
+ return r.ThrottleTime
+}
diff --git a/vendor/github.com/IBM/sarama/utils.go b/vendor/github.com/IBM/sarama/utils.go
new file mode 100644
index 0000000..9b87cb8
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/utils.go
@@ -0,0 +1,393 @@
+package sarama
+
+import (
+ "bufio"
+ "fmt"
+ "math/rand"
+ "net"
+ "regexp"
+ "time"
+)
+
+const (
+ defaultRetryBackoff = 100 * time.Millisecond
+ defaultRetryMaxBackoff = 1000 * time.Millisecond
+)
+
+type none struct{}
+
+// make []int32 sortable so we can sort partition numbers
+type int32Slice []int32
+
+func (slice int32Slice) Len() int {
+ return len(slice)
+}
+
+func (slice int32Slice) Less(i, j int) bool {
+ return slice[i] < slice[j]
+}
+
+func (slice int32Slice) Swap(i, j int) {
+ slice[i], slice[j] = slice[j], slice[i]
+}
+
+func dupInt32Slice(input []int32) []int32 {
+ ret := make([]int32, 0, len(input))
+ ret = append(ret, input...)
+ return ret
+}
+
+func withRecover(fn func()) {
+ defer func() {
+ handler := PanicHandler
+ if handler != nil {
+ if err := recover(); err != nil {
+ handler(err)
+ }
+ }
+ }()
+
+ fn()
+}
+
+func safeAsyncClose(b *Broker) {
+ go withRecover(func() {
+ if connected, _ := b.Connected(); connected {
+ if err := b.Close(); err != nil {
+ Logger.Println("Error closing broker", b.ID(), ":", err)
+ }
+ }
+ })
+}
+
+// Encoder is a simple interface for any type that can be encoded as an array of bytes
+// in order to be sent as the key or value of a Kafka message. Length() is provided as an
+// optimization, and must return the same as len() on the result of Encode().
+type Encoder interface {
+ Encode() ([]byte, error)
+ Length() int
+}
+
+// make strings and byte slices encodable for convenience so they can be used as keys
+// and/or values in kafka messages
+
+// StringEncoder implements the Encoder interface for Go strings so that they can be used
+// as the Key or Value in a ProducerMessage.
+type StringEncoder string
+
+func (s StringEncoder) Encode() ([]byte, error) {
+ return []byte(s), nil
+}
+
+func (s StringEncoder) Length() int {
+ return len(s)
+}
+
+// ByteEncoder implements the Encoder interface for Go byte slices so that they can be used
+// as the Key or Value in a ProducerMessage.
+type ByteEncoder []byte
+
+func (b ByteEncoder) Encode() ([]byte, error) {
+ return b, nil
+}
+
+func (b ByteEncoder) Length() int {
+ return len(b)
+}
+
+// bufConn wraps a net.Conn with a buffer for reads to reduce the number of
+// reads that trigger syscalls.
+type bufConn struct {
+ net.Conn
+ buf *bufio.Reader
+}
+
+func newBufConn(conn net.Conn) *bufConn {
+ return &bufConn{
+ Conn: conn,
+ buf: bufio.NewReader(conn),
+ }
+}
+
+func (bc *bufConn) Read(b []byte) (n int, err error) {
+ return bc.buf.Read(b)
+}
+
+// KafkaVersion instances represent versions of the upstream Kafka broker.
+type KafkaVersion struct {
+ // it's a struct rather than just typing the array directly to make it opaque and stop people
+ // generating their own arbitrary versions
+ version [4]uint
+}
+
+func newKafkaVersion(major, minor, veryMinor, patch uint) KafkaVersion {
+ return KafkaVersion{
+ version: [4]uint{major, minor, veryMinor, patch},
+ }
+}
+
+// IsAtLeast return true if and only if the version it is called on is
+// greater than or equal to the version passed in:
+//
+// V1.IsAtLeast(V2) // false
+// V2.IsAtLeast(V1) // true
+func (v KafkaVersion) IsAtLeast(other KafkaVersion) bool {
+ for i := range v.version {
+ if v.version[i] > other.version[i] {
+ return true
+ } else if v.version[i] < other.version[i] {
+ return false
+ }
+ }
+ return true
+}
+
+// Effective constants defining the supported kafka versions.
+var (
+ V0_8_2_0 = newKafkaVersion(0, 8, 2, 0)
+ V0_8_2_1 = newKafkaVersion(0, 8, 2, 1)
+ V0_8_2_2 = newKafkaVersion(0, 8, 2, 2)
+ V0_9_0_0 = newKafkaVersion(0, 9, 0, 0)
+ V0_9_0_1 = newKafkaVersion(0, 9, 0, 1)
+ V0_10_0_0 = newKafkaVersion(0, 10, 0, 0)
+ V0_10_0_1 = newKafkaVersion(0, 10, 0, 1)
+ V0_10_1_0 = newKafkaVersion(0, 10, 1, 0)
+ V0_10_1_1 = newKafkaVersion(0, 10, 1, 1)
+ V0_10_2_0 = newKafkaVersion(0, 10, 2, 0)
+ V0_10_2_1 = newKafkaVersion(0, 10, 2, 1)
+ V0_10_2_2 = newKafkaVersion(0, 10, 2, 2)
+ V0_11_0_0 = newKafkaVersion(0, 11, 0, 0)
+ V0_11_0_1 = newKafkaVersion(0, 11, 0, 1)
+ V0_11_0_2 = newKafkaVersion(0, 11, 0, 2)
+ V1_0_0_0 = newKafkaVersion(1, 0, 0, 0)
+ V1_0_1_0 = newKafkaVersion(1, 0, 1, 0)
+ V1_0_2_0 = newKafkaVersion(1, 0, 2, 0)
+ V1_1_0_0 = newKafkaVersion(1, 1, 0, 0)
+ V1_1_1_0 = newKafkaVersion(1, 1, 1, 0)
+ V2_0_0_0 = newKafkaVersion(2, 0, 0, 0)
+ V2_0_1_0 = newKafkaVersion(2, 0, 1, 0)
+ V2_1_0_0 = newKafkaVersion(2, 1, 0, 0)
+ V2_1_1_0 = newKafkaVersion(2, 1, 1, 0)
+ V2_2_0_0 = newKafkaVersion(2, 2, 0, 0)
+ V2_2_1_0 = newKafkaVersion(2, 2, 1, 0)
+ V2_2_2_0 = newKafkaVersion(2, 2, 2, 0)
+ V2_3_0_0 = newKafkaVersion(2, 3, 0, 0)
+ V2_3_1_0 = newKafkaVersion(2, 3, 1, 0)
+ V2_4_0_0 = newKafkaVersion(2, 4, 0, 0)
+ V2_4_1_0 = newKafkaVersion(2, 4, 1, 0)
+ V2_5_0_0 = newKafkaVersion(2, 5, 0, 0)
+ V2_5_1_0 = newKafkaVersion(2, 5, 1, 0)
+ V2_6_0_0 = newKafkaVersion(2, 6, 0, 0)
+ V2_6_1_0 = newKafkaVersion(2, 6, 1, 0)
+ V2_6_2_0 = newKafkaVersion(2, 6, 2, 0)
+ V2_6_3_0 = newKafkaVersion(2, 6, 3, 0)
+ V2_7_0_0 = newKafkaVersion(2, 7, 0, 0)
+ V2_7_1_0 = newKafkaVersion(2, 7, 1, 0)
+ V2_7_2_0 = newKafkaVersion(2, 7, 2, 0)
+ V2_8_0_0 = newKafkaVersion(2, 8, 0, 0)
+ V2_8_1_0 = newKafkaVersion(2, 8, 1, 0)
+ V2_8_2_0 = newKafkaVersion(2, 8, 2, 0)
+ V3_0_0_0 = newKafkaVersion(3, 0, 0, 0)
+ V3_0_1_0 = newKafkaVersion(3, 0, 1, 0)
+ V3_0_2_0 = newKafkaVersion(3, 0, 2, 0)
+ V3_1_0_0 = newKafkaVersion(3, 1, 0, 0)
+ V3_1_1_0 = newKafkaVersion(3, 1, 1, 0)
+ V3_1_2_0 = newKafkaVersion(3, 1, 2, 0)
+ V3_2_0_0 = newKafkaVersion(3, 2, 0, 0)
+ V3_2_1_0 = newKafkaVersion(3, 2, 1, 0)
+ V3_2_2_0 = newKafkaVersion(3, 2, 2, 0)
+ V3_2_3_0 = newKafkaVersion(3, 2, 3, 0)
+ V3_3_0_0 = newKafkaVersion(3, 3, 0, 0)
+ V3_3_1_0 = newKafkaVersion(3, 3, 1, 0)
+ V3_3_2_0 = newKafkaVersion(3, 3, 2, 0)
+ V3_4_0_0 = newKafkaVersion(3, 4, 0, 0)
+ V3_4_1_0 = newKafkaVersion(3, 4, 1, 0)
+ V3_5_0_0 = newKafkaVersion(3, 5, 0, 0)
+ V3_5_1_0 = newKafkaVersion(3, 5, 1, 0)
+ V3_5_2_0 = newKafkaVersion(3, 5, 2, 0)
+ V3_6_0_0 = newKafkaVersion(3, 6, 0, 0)
+ V3_6_1_0 = newKafkaVersion(3, 6, 1, 0)
+ V3_6_2_0 = newKafkaVersion(3, 6, 2, 0)
+ V3_7_0_0 = newKafkaVersion(3, 7, 0, 0)
+ V3_7_1_0 = newKafkaVersion(3, 7, 1, 0)
+ V3_7_2_0 = newKafkaVersion(3, 7, 2, 0)
+ V3_8_0_0 = newKafkaVersion(3, 8, 0, 0)
+ V3_8_1_0 = newKafkaVersion(3, 8, 1, 0)
+ V3_9_0_0 = newKafkaVersion(3, 9, 0, 0)
+ V3_9_1_0 = newKafkaVersion(3, 9, 1, 0)
+ V4_0_0_0 = newKafkaVersion(4, 0, 0, 0)
+ V4_1_0_0 = newKafkaVersion(4, 1, 0, 0)
+
+ SupportedVersions = []KafkaVersion{
+ V0_8_2_0,
+ V0_8_2_1,
+ V0_8_2_2,
+ V0_9_0_0,
+ V0_9_0_1,
+ V0_10_0_0,
+ V0_10_0_1,
+ V0_10_1_0,
+ V0_10_1_1,
+ V0_10_2_0,
+ V0_10_2_1,
+ V0_10_2_2,
+ V0_11_0_0,
+ V0_11_0_1,
+ V0_11_0_2,
+ V1_0_0_0,
+ V1_0_1_0,
+ V1_0_2_0,
+ V1_1_0_0,
+ V1_1_1_0,
+ V2_0_0_0,
+ V2_0_1_0,
+ V2_1_0_0,
+ V2_1_1_0,
+ V2_2_0_0,
+ V2_2_1_0,
+ V2_2_2_0,
+ V2_3_0_0,
+ V2_3_1_0,
+ V2_4_0_0,
+ V2_4_1_0,
+ V2_5_0_0,
+ V2_5_1_0,
+ V2_6_0_0,
+ V2_6_1_0,
+ V2_6_2_0,
+ V2_6_3_0,
+ V2_7_0_0,
+ V2_7_1_0,
+ V2_7_2_0,
+ V2_8_0_0,
+ V2_8_1_0,
+ V2_8_2_0,
+ V3_0_0_0,
+ V3_0_1_0,
+ V3_0_2_0,
+ V3_1_0_0,
+ V3_1_1_0,
+ V3_1_2_0,
+ V3_2_0_0,
+ V3_2_1_0,
+ V3_2_2_0,
+ V3_2_3_0,
+ V3_3_0_0,
+ V3_3_1_0,
+ V3_3_2_0,
+ V3_4_0_0,
+ V3_4_1_0,
+ V3_5_0_0,
+ V3_5_1_0,
+ V3_5_2_0,
+ V3_6_0_0,
+ V3_6_1_0,
+ V3_6_2_0,
+ V3_7_0_0,
+ V3_7_1_0,
+ V3_7_2_0,
+ V3_8_0_0,
+ V3_8_1_0,
+ V3_9_0_0,
+ V3_9_1_0,
+ V4_0_0_0,
+ V4_1_0_0,
+ }
+ MinVersion = V0_8_2_0
+ MaxVersion = V4_1_0_0
+ DefaultVersion = V2_1_0_0
+
+ // reduced set of protocol versions to matrix test
+ fvtRangeVersions = []KafkaVersion{
+ V0_8_2_2,
+ V0_10_2_2,
+ V1_0_2_0,
+ V1_1_1_0,
+ V2_0_1_0,
+ V2_2_2_0,
+ V2_4_1_0,
+ V2_6_3_0,
+ V2_8_2_0,
+ V3_1_2_0,
+ V3_3_2_0,
+ V3_6_2_0,
+ }
+)
+
+var (
+ // This regex validates that a string complies with the pre kafka 1.0.0 format for version strings, for example 0.11.0.3
+ validPreKafka1Version = regexp.MustCompile(`^0\.\d+\.\d+\.\d+$`)
+
+ // This regex validates that a string complies with the post Kafka 1.0.0 format, for example 1.0.0
+ validPostKafka1Version = regexp.MustCompile(`^\d+\.\d+\.\d+$`)
+)
+
+// ParseKafkaVersion parses and returns kafka version or error from a string
+func ParseKafkaVersion(s string) (KafkaVersion, error) {
+ if len(s) < 5 {
+ return DefaultVersion, fmt.Errorf("invalid version `%s`", s)
+ }
+ var major, minor, veryMinor, patch uint
+ var err error
+ if s[0] == '0' {
+ err = scanKafkaVersion(s, validPreKafka1Version, "0.%d.%d.%d", [3]*uint{&minor, &veryMinor, &patch})
+ } else {
+ err = scanKafkaVersion(s, validPostKafka1Version, "%d.%d.%d", [3]*uint{&major, &minor, &veryMinor})
+ }
+ if err != nil {
+ return DefaultVersion, err
+ }
+ return newKafkaVersion(major, minor, veryMinor, patch), nil
+}
+
+func scanKafkaVersion(s string, pattern *regexp.Regexp, format string, v [3]*uint) error {
+ if !pattern.MatchString(s) {
+ return fmt.Errorf("invalid version `%s`", s)
+ }
+ _, err := fmt.Sscanf(s, format, v[0], v[1], v[2])
+ return err
+}
+
+func (v KafkaVersion) String() string {
+ if v.version[0] == 0 {
+ return fmt.Sprintf("0.%d.%d.%d", v.version[1], v.version[2], v.version[3])
+ }
+
+ return fmt.Sprintf("%d.%d.%d", v.version[0], v.version[1], v.version[2])
+}
+
+// NewExponentialBackoff returns a function that implements an exponential backoff strategy with jitter.
+// It follows KIP-580, implementing the formula:
+// MIN(retry.backoff.max.ms, (retry.backoff.ms * 2**(failures - 1)) * random(0.8, 1.2))
+// This ensures retries start with `backoff` and exponentially increase until `maxBackoff`, with added jitter.
+// The behavior when `failures = 0` is not explicitly defined in KIP-580 and is left to implementation discretion.
+//
+// Example usage:
+//
+// backoffFunc := sarama.NewExponentialBackoff(config.Producer.Retry.Backoff, 2*time.Second)
+// config.Producer.Retry.BackoffFunc = backoffFunc
+func NewExponentialBackoff(backoff time.Duration, maxBackoff time.Duration) func(retries, maxRetries int) time.Duration {
+ if backoff <= 0 {
+ backoff = defaultRetryBackoff
+ }
+ if maxBackoff <= 0 {
+ maxBackoff = defaultRetryMaxBackoff
+ }
+
+ if backoff > maxBackoff {
+ Logger.Println("Warning: backoff is greater than maxBackoff, using maxBackoff instead.")
+ backoff = maxBackoff
+ }
+
+ return func(retries, maxRetries int) time.Duration {
+ if retries <= 0 {
+ return backoff
+ }
+
+ calculatedBackoff := backoff * time.Duration(1<<(retries-1))
+ jitter := 0.8 + 0.4*rand.Float64()
+ calculatedBackoff = time.Duration(float64(calculatedBackoff) * jitter)
+
+ return min(calculatedBackoff, maxBackoff)
+ }
+}
diff --git a/vendor/github.com/IBM/sarama/version.go b/vendor/github.com/IBM/sarama/version.go
new file mode 100644
index 0000000..d3b9d53
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/version.go
@@ -0,0 +1,27 @@
+package sarama
+
+import (
+ "runtime/debug"
+ "sync"
+)
+
+var (
+ v string
+ vOnce sync.Once
+)
+
+func version() string {
+ vOnce.Do(func() {
+ bi, ok := debug.ReadBuildInfo()
+ if ok {
+ v = bi.Main.Version
+ }
+ if v == "" || v == "(devel)" {
+ // if we can't read a go module version then they're using a git
+ // clone or vendored module so all we can do is report "dev" for
+ // the version to make a valid ApiVersions request
+ v = "dev"
+ }
+ })
+ return v
+}
diff --git a/vendor/github.com/IBM/sarama/zstd.go b/vendor/github.com/IBM/sarama/zstd.go
new file mode 100644
index 0000000..6073ce7
--- /dev/null
+++ b/vendor/github.com/IBM/sarama/zstd.go
@@ -0,0 +1,74 @@
+package sarama
+
+import (
+ "sync"
+
+ "github.com/klauspost/compress/zstd"
+)
+
+// zstdMaxBufferedEncoders maximum number of not-in-use zstd encoders
+// If the pool of encoders is exhausted then new encoders will be created on the fly
+const zstdMaxBufferedEncoders = 1
+
+type ZstdEncoderParams struct {
+ Level int
+}
+type ZstdDecoderParams struct {
+}
+
+var zstdDecMap sync.Map
+
+var zstdAvailableEncoders sync.Map
+
+func getZstdEncoderChannel(params ZstdEncoderParams) chan *zstd.Encoder {
+ if c, ok := zstdAvailableEncoders.Load(params); ok {
+ return c.(chan *zstd.Encoder)
+ }
+ c, _ := zstdAvailableEncoders.LoadOrStore(params, make(chan *zstd.Encoder, zstdMaxBufferedEncoders))
+ return c.(chan *zstd.Encoder)
+}
+
+func getZstdEncoder(params ZstdEncoderParams) *zstd.Encoder {
+ select {
+ case enc := <-getZstdEncoderChannel(params):
+ return enc
+ default:
+ encoderLevel := zstd.SpeedDefault
+ if params.Level != CompressionLevelDefault {
+ encoderLevel = zstd.EncoderLevelFromZstd(params.Level)
+ }
+ zstdEnc, _ := zstd.NewWriter(nil, zstd.WithZeroFrames(true),
+ zstd.WithEncoderLevel(encoderLevel),
+ zstd.WithEncoderConcurrency(1))
+ return zstdEnc
+ }
+}
+
+func releaseEncoder(params ZstdEncoderParams, enc *zstd.Encoder) {
+ select {
+ case getZstdEncoderChannel(params) <- enc:
+ default:
+ }
+}
+
+func getDecoder(params ZstdDecoderParams) *zstd.Decoder {
+ if ret, ok := zstdDecMap.Load(params); ok {
+ return ret.(*zstd.Decoder)
+ }
+ // It's possible to race and create multiple new readers.
+ // Only one will survive GC after use.
+ zstdDec, _ := zstd.NewReader(nil, zstd.WithDecoderConcurrency(0))
+ zstdDecMap.Store(params, zstdDec)
+ return zstdDec
+}
+
+func zstdDecompress(params ZstdDecoderParams, dst, src []byte) ([]byte, error) {
+ return getDecoder(params).DecodeAll(src, dst)
+}
+
+func zstdCompress(params ZstdEncoderParams, dst, src []byte) ([]byte, error) {
+ enc := getZstdEncoder(params)
+ out := enc.EncodeAll(src, dst)
+ releaseEncoder(params, enc)
+ return out, nil
+}