Module: Kafka::FFI

Extended by:

FFI::Library

Defined in:

lib/kafka/ffi.rb,
lib/kafka/ffi/admin.rb,
lib/kafka/ffi/error.rb,
lib/kafka/ffi/event.rb,
lib/kafka/ffi/queue.rb,
lib/kafka/ffi/topic.rb,
lib/kafka/ffi/client.rb,
lib/kafka/ffi/config.rb,
lib/kafka/ffi/opaque.rb,
lib/kafka/ffi/message.rb,
lib/kafka/ffi/consumer.rb,
lib/kafka/ffi/metadata.rb,
lib/kafka/ffi/producer.rb,
lib/kafka/ffi/group_info.rb,
lib/kafka/ffi/group_list.rb,
lib/kafka/ffi/topic_config.rb,
lib/kafka/ffi/message/header.rb,
lib/kafka/ffi/opaque_pointer.rb,
lib/kafka/ffi/topic_metadata.rb,
lib/kafka/ffi/broker_metadata.rb,
lib/kafka/ffi/topic_partition.rb,
lib/kafka/ffi/group_member_info.rb,
lib/kafka/ffi/partition_metadata.rb,
lib/kafka/ffi/topic_partition_list.rb

Overview

Module FFI provides both a (mostly) complete set of low level function calls into librdkafka as well as a set of slightly higher level abstractions and objects that make working with the API easier. It is still required to know enough about using librdkafka to use the abstractions safely (see the introduction below).

All exposed functions on Kafka::FFI are named to match the functions exposed by librdkafka (see rdkafka.h).

See: github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md See: github.com/edenhill/librdkafka/blob/master/src/rdkafka.h

Defined Under Namespace

Modules: Admin Classes: BrokerMetadata, Client, Config, ConfigError, Consumer, Error, Event, GroupInfo, GroupList, GroupMemberInfo, InvalidConfigValue, Message, Metadata, Opaque, OpaquePointer, PartitionMetadata, Producer, Queue, Topic, TopicAlreadyConfiguredError, TopicConfig, TopicMetadata, TopicPartition, TopicPartitionList, UnknownConfigKey

Constant Summary

RD_KAFKA_RESP_ERR__BEGIN =

Response Errors

-200

RD_KAFKA_RESP_ERR__BAD_MSG =

-199

RD_KAFKA_RESP_ERR__BAD_COMPRESSION =

-198

RD_KAFKA_RESP_ERR__DESTROY =

-197

RD_KAFKA_RESP_ERR__FAIL =

-196

RD_KAFKA_RESP_ERR__TRANSPORT =

-195

RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE =

-194

RD_KAFKA_RESP_ERR__RESOLVE =

-193

RD_KAFKA_RESP_ERR__MSG_TIMED_OUT =

-192

RD_KAFKA_RESP_ERR__PARTITION_EOF =

-191

RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION =

-190

RD_KAFKA_RESP_ERR__FS =

-189

RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC =

-188

RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN =

-187

RD_KAFKA_RESP_ERR__INVALID_ARG =

-186

RD_KAFKA_RESP_ERR__TIMED_OUT =

-185

RD_KAFKA_RESP_ERR__QUEUE_FULL =

-184

RD_KAFKA_RESP_ERR__ISR_INSUFF =

-183

RD_KAFKA_RESP_ERR__NODE_UPDATE =

-182

RD_KAFKA_RESP_ERR__SSL =

-181

RD_KAFKA_RESP_ERR__WAIT_COORD =

-180

RD_KAFKA_RESP_ERR__UNKNOWN_GROUP =

-179

RD_KAFKA_RESP_ERR__IN_PROGRESS =

-178

RD_KAFKA_RESP_ERR__PREV_IN_PROGRESS =

-177

RD_KAFKA_RESP_ERR__EXISTING_SUBSCRIPTION =

-176

RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS =

-175

RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS =

-174

RD_KAFKA_RESP_ERR__CONFLICT =

-173

RD_KAFKA_RESP_ERR__STATE =

-172

RD_KAFKA_RESP_ERR__UNKNOWN_PROTOCOL =

-171

RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED =

-170

RD_KAFKA_RESP_ERR__AUTHENTICATION =

-169

RD_KAFKA_RESP_ERR__NO_OFFSET =

-168

RD_KAFKA_RESP_ERR__OUTDATED =

-167

RD_KAFKA_RESP_ERR__TIMED_OUT_QUEUE =

-166

RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE =

-165

RD_KAFKA_RESP_ERR__WAIT_CACHE =

-164

RD_KAFKA_RESP_ERR__INTR =

-163

RD_KAFKA_RESP_ERR__KEY_SERIALIZATION =

-162

RD_KAFKA_RESP_ERR__VALUE_SERIALIZATION =

-161

RD_KAFKA_RESP_ERR__KEY_DESERIALIZATION =

-160

RD_KAFKA_RESP_ERR__VALUE_DESERIALIZATION =

-159

RD_KAFKA_RESP_ERR__PARTIAL =

-158

RD_KAFKA_RESP_ERR__READ_ONLY =

-157

RD_KAFKA_RESP_ERR__NOENT =

-156

RD_KAFKA_RESP_ERR__UNDERFLOW =

-155

RD_KAFKA_RESP_ERR__INVALID_TYPE =

-154

RD_KAFKA_RESP_ERR__RETRY =

-153

RD_KAFKA_RESP_ERR__PURGE_QUEUE =

-152

RD_KAFKA_RESP_ERR__PURGE_INFLIGHT =

-151

RD_KAFKA_RESP_ERR__FATAL =

-150

RD_KAFKA_RESP_ERR__INCONSISTENT =

-149

RD_KAFKA_RESP_ERR__GAPLESS_GUARANTEE =

-148

RD_KAFKA_RESP_ERR__MAX_POLL_EXCEEDED =

-147

RD_KAFKA_RESP_ERR__UNKNOWN_BROKER =

-146

RD_KAFKA_RESP_ERR__END =

-100

RD_KAFKA_RESP_ERR_UNKNOWN =

-1

RD_KAFKA_RESP_ERR_NO_ERROR =

0

RD_KAFKA_RESP_ERR_OFFSET_OUT_OF_RANGE =

1

RD_KAFKA_RESP_ERR_INVALID_MSG =

2

RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART =

3

RD_KAFKA_RESP_ERR_INVALID_MSG_SIZE =

4

RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE =

5

RD_KAFKA_RESP_ERR_NOT_LEADER_FOR_PARTITION =

6

RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT =

7

RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE =

8

RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE =

9

RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE =

10

RD_KAFKA_RESP_ERR_STALE_CTRL_EPOCH =

11

RD_KAFKA_RESP_ERR_OFFSET_METADATA_TOO_LARGE =

12

RD_KAFKA_RESP_ERR_NETWORK_EXCEPTION =

13

RD_KAFKA_RESP_ERR_COORDINATOR_LOAD_IN_PROGRESS =

14

RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE =

15

RD_KAFKA_RESP_ERR_NOT_COORDINATOR =

16

RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION =

17

RD_KAFKA_RESP_ERR_RECORD_LIST_TOO_LARGE =

18

RD_KAFKA_RESP_ERR_NOT_ENOUGH_REPLICAS =

19

RD_KAFKA_RESP_ERR_NOT_ENOUGH_REPLICAS_AFTER_APPEND =

20

RD_KAFKA_RESP_ERR_INVALID_REQUIRED_ACKS =

21

RD_KAFKA_RESP_ERR_ILLEGAL_GENERATION =

22

RD_KAFKA_RESP_ERR_INCONSISTENT_GROUP_PROTOCOL =

23

RD_KAFKA_RESP_ERR_INVALID_GROUP_ID =

24

RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID =

25

RD_KAFKA_RESP_ERR_INVALID_SESSION_TIMEOUT =

26

RD_KAFKA_RESP_ERR_REBALANCE_IN_PROGRESS =

27

RD_KAFKA_RESP_ERR_INVALID_COMMIT_OFFSET_SIZE =

28

RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED =

29

RD_KAFKA_RESP_ERR_GROUP_AUTHORIZATION_FAILED =

30

RD_KAFKA_RESP_ERR_CLUSTER_AUTHORIZATION_FAILED =

31

RD_KAFKA_RESP_ERR_INVALID_TIMESTAMP =

32

RD_KAFKA_RESP_ERR_UNSUPPORTED_SASL_MECHANISM =

33

RD_KAFKA_RESP_ERR_ILLEGAL_SASL_STATE =

34

RD_KAFKA_RESP_ERR_UNSUPPORTED_VERSION =

35

RD_KAFKA_RESP_ERR_TOPIC_ALREADY_EXISTS =

36

RD_KAFKA_RESP_ERR_INVALID_PARTITIONS =

37

RD_KAFKA_RESP_ERR_INVALID_REPLICATION_FACTOR =

38

RD_KAFKA_RESP_ERR_INVALID_REPLICA_ASSIGNMENT =

39

RD_KAFKA_RESP_ERR_INVALID_CONFIG =

40

RD_KAFKA_RESP_ERR_NOT_CONTROLLER =

41

RD_KAFKA_RESP_ERR_INVALID_REQUEST =

42

RD_KAFKA_RESP_ERR_UNSUPPORTED_FOR_MESSAGE_FORMAT =

43

RD_KAFKA_RESP_ERR_POLICY_VIOLATION =

44

RD_KAFKA_RESP_ERR_OUT_OF_ORDER_SEQUENCE_NUMBER =

45

RD_KAFKA_RESP_ERR_DUPLICATE_SEQUENCE_NUMBER =

46

RD_KAFKA_RESP_ERR_INVALID_PRODUCER_EPOCH =

47

RD_KAFKA_RESP_ERR_INVALID_TXN_STATE =

48

RD_KAFKA_RESP_ERR_INVALID_PRODUCER_ID_MAPPING =

49

RD_KAFKA_RESP_ERR_INVALID_TRANSACTION_TIMEOUT =

50

RD_KAFKA_RESP_ERR_CONCURRENT_TRANSACTIONS =

51

RD_KAFKA_RESP_ERR_TRANSACTION_COORDINATOR_FENCED =

52

RD_KAFKA_RESP_ERR_TRANSACTIONAL_ID_AUTHORIZATION_FAILED =

53

RD_KAFKA_RESP_ERR_SECURITY_DISABLED =

54

RD_KAFKA_RESP_ERR_OPERATION_NOT_ATTEMPTED =

55

RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR =

56

RD_KAFKA_RESP_ERR_LOG_DIR_NOT_FOUND =

57

RD_KAFKA_RESP_ERR_SASL_AUTHENTICATION_FAILED =

58

RD_KAFKA_RESP_ERR_UNKNOWN_PRODUCER_ID =

59

RD_KAFKA_RESP_ERR_REASSIGNMENT_IN_PROGRESS =

60

RD_KAFKA_RESP_ERR_DELEGATION_TOKEN_AUTH_DISABLED =

61

RD_KAFKA_RESP_ERR_DELEGATION_TOKEN_NOT_FOUND =

62

RD_KAFKA_RESP_ERR_DELEGATION_TOKEN_OWNER_MISMATCH =

63

RD_KAFKA_RESP_ERR_DELEGATION_TOKEN_REQUEST_NOT_ALLOWED =

64

RD_KAFKA_RESP_ERR_DELEGATION_TOKEN_AUTHORIZATION_FAILED =

65

RD_KAFKA_RESP_ERR_DELEGATION_TOKEN_EXPIRED =

66

RD_KAFKA_RESP_ERR_INVALID_PRINCIPAL_TYPE =

67

RD_KAFKA_RESP_ERR_NON_EMPTY_GROUP =

68

RD_KAFKA_RESP_ERR_GROUP_ID_NOT_FOUND =

69

RD_KAFKA_RESP_ERR_FETCH_SESSION_ID_NOT_FOUND =

70

RD_KAFKA_RESP_ERR_INVALID_FETCH_SESSION_EPOCH =

71

RD_KAFKA_RESP_ERR_LISTENER_NOT_FOUND =

72

RD_KAFKA_RESP_ERR_TOPIC_DELETION_DISABLED =

73

RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH =

74

RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH =

75

RD_KAFKA_RESP_ERR_UNSUPPORTED_COMPRESSION_TYPE =

76

RD_KAFKA_RESP_ERR_STALE_BROKER_EPOCH =

77

RD_KAFKA_RESP_ERR_OFFSET_NOT_AVAILABLE =

78

RD_KAFKA_RESP_ERR_MEMBER_ID_REQUIRED =

79

RD_KAFKA_RESP_ERR_PREFERRED_LEADER_NOT_AVAILABLE =

80

RD_KAFKA_RESP_ERR_GROUP_MAX_SIZE_REACHED =

81

RD_KAFKA_OFFSET_BEGINNING =

-2

RD_KAFKA_OFFSET_END =

-1

RD_KAFKA_OFFSET_STORED =

-1000

RD_KAFKA_OFFSET_INVALID =

-1001

RD_KAFKA_EVENT_NONE =

0x00

RD_KAFKA_EVENT_DR =

0x01

RD_KAFKA_EVENT_FETCH =

0x02

RD_KAFKA_EVENT_LOG =

0x04

RD_KAFKA_EVENT_ERROR =

0x08

RD_KAFKA_EVENT_REBALANCE =

0x10

RD_KAFKA_EVENT_OFFSET_COMMIT =

0x20

RD_KAFKA_EVENT_STATS =

0x40

RD_KAFKA_EVENT_CREATETOPICS_RESULT =

100

RD_KAFKA_EVENT_DELETETOPICS_RESULT =

101

RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT =

102

RD_KAFKA_EVENT_ALTERCONFIGS_RESULT =

103

RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT =

104

RD_KAFKA_EVENT_OAUTHBEARER_TOKEN_REFRESH =

0x100

RD_KAFKA_MSG_STATUS_NOT_PERSISTED =

0

RD_KAFKA_MSG_STATUS_POSSIBLY_PERSISTED =

1

RD_KAFKA_MSG_STATUS_PERSISTED =

2

RD_KAFKA_MSG_F_FREE =

Flags for rd_kafka_produce, rd_kafka_producev, and rd_kafka_produce_batch.

See Also:

• Kafka::FFI.rdkafkardkafka.h

0x01

RD_KAFKA_MSG_F_COPY =

0x02

RD_KAFKA_MSG_F_BLOCK =

0x04

RD_KAFKA_MSG_F_PARTITION =

0x04

RD_KAFKA_PURGE_F_QUEUE =

Flags for rd_kafka_purge

See Also:

• Kafka::FFI.rdkafkardkafka.h

0x01

RD_KAFKA_PURGE_F_INFLIGHT =

0x02

RD_KAFKA_PURGE_F_NON_BLOCKING =

0x04

RD_KAFKA_VTYPE_END =

rd_kafka_producev va-arg vtype constants.

0

RD_KAFKA_VTYPE_TOPIC =

1

RD_KAFKA_VTYPE_RKT =

2

RD_KAFKA_VTYPE_PARTITION =

3

RD_KAFKA_VTYPE_VALUE =

4

RD_KAFKA_VTYPE_KEY =

5

RD_KAFKA_VTYPE_OPAQUE =

6

RD_KAFKA_VTYPE_MSGFLAGS =

7

RD_KAFKA_VTYPE_TIMESTAMP =

8

RD_KAFKA_VTYPE_HEADER =

9

RD_KAFKA_VTYPE_HEADERS =

10

RD_KAFKA_PARTITION_UA =

Use for partition when it should be assigned by the configured partitioner.

-1

RD_KAFKA_ADMIN_OP_ANY =

0

RD_KAFKA_ADMIN_OP_CREATETOPICS =

1

RD_KAFKA_ADMIN_OP_DELETETOPICS =

2

RD_KAFKA_ADMIN_OP_CREATEPARTITIONS =

3

RD_KAFKA_ADMIN_OP_ALTERCONFIGS =

4

RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS =

5

RD_KAFKA_RESOURCE_UNKNOWN =

0

RD_KAFKA_RESOURCE_ANY =

1

RD_KAFKA_RESOURCE_TOPIC =

2

RD_KAFKA_RESOURCE_GROUP =

3

RD_KAFKA_RESOURCE_BROKER =

4

RD_KAFKA_CONFIG_SOURCE_UNKNOWN_CONFIG =

0

RD_KAFKA_CONFIG_SOURCE_DYNAMIC_TOPIC_CONFIG =

1

RD_KAFKA_CONFIG_SOURCE_DYNAMIC_BROKER_CONFIG =

2

RD_KAFKA_CONFIG_SOURCE_DYNAMIC_DEFAULT_BROKER_CONFIG =

3

RD_KAFKA_CONFIG_SOURCE_STATIC_BROKER_CONFIG =

4

RD_KAFKA_CONFIG_SOURCE_DEFAULT_CONFIG =

5

Class Method Summary

• .features ⇒ Array<String>

  Returns the set of features available from librdkafka.

• .version ⇒ String

  Returns the loaded version of librdkafka.

Instance Method Summary

• #rd_kafka_AdminOptions_new ⇒ Object

  AdminOptions.

• #rd_kafka_AlterConfigs ⇒ Object

  AlterConfigs.

• #rd_kafka_ConfigEntry_name ⇒ Object

  ConfigEntry.

• #rd_kafka_ConfigResource_new ⇒ Object

  ConfigResource.

• #rd_kafka_ConfigSource_name ⇒ Object

  ConfigSource.

• #rd_kafka_consume_start ⇒ Object

  Legacy Simple Consumer API.

• #rd_kafka_CreateTopics ⇒ Object

  Create Topics / NewTopic.

• #rd_kafka_DescribeConfigs ⇒ Object

  DescribeConfigs.

• #rd_kafka_msg_partitioner_random ⇒ Object

  Partitioners.

• #rd_kafka_ResourceType_name ⇒ Object

  Resource Type (enum).

• #rd_kafka_subscribe ⇒ Object

  High Level Consumer API.

Class Method Details

.features ⇒ Array<String>

Returns the set of features available from librdkafka

Returns:

• (Array<String>) —

  List of available features

# File 'lib/kafka/ffi.rb', line 36

def self.features
  conf = Config.new
  conf.get("builtin.features").split(",")
ensure
  conf.destroy
end

.version ⇒ String

Returns the loaded version of librdkafka

Returns:

• (String) —

  Version string

# File 'lib/kafka/ffi.rb', line 29

def self.version
  rd_kafka_version_str
end

Instance Method Details

#rd_kafka_AdminOptions_new ⇒ Object

AdminOptions

# File 'lib/kafka/ffi.rb', line 690

attach_function :rd_kafka_AdminOptions_new, [Client, :admin_operation], Admin::AdminOptions

#rd_kafka_AlterConfigs ⇒ Object

AlterConfigs

# File 'lib/kafka/ffi.rb', line 703

attach_function :rd_kafka_AlterConfigs, [Client, :pointer, :size_t, Admin::AdminOptions, Queue], :void

#rd_kafka_ConfigEntry_name ⇒ Object

ConfigEntry

# File 'lib/kafka/ffi.rb', line 721

attach_function :rd_kafka_ConfigEntry_name, [Admin::ConfigEntry], :string

#rd_kafka_ConfigResource_new ⇒ Object

ConfigResource

# File 'lib/kafka/ffi.rb', line 710

attach_function :rd_kafka_ConfigResource_new, [:resource_type, :string], Admin::ConfigResource

#rd_kafka_ConfigSource_name ⇒ Object

ConfigSource

# File 'lib/kafka/ffi.rb', line 731

attach_function :rd_kafka_ConfigSource_name, [:config_source], :string

#rd_kafka_consume_start ⇒ Object

Legacy Simple Consumer API

# File 'lib/kafka/ffi.rb', line 573

attach_function :rd_kafka_consume_start, [Topic, :partition, :offset], :int

#rd_kafka_CreateTopics ⇒ Object

Create Topics / NewTopic

# File 'lib/kafka/ffi.rb', line 734

attach_function :rd_kafka_CreateTopics, [Client, :pointer, :size_t, Admin::AdminOptions, Queue], :void, blocking: true

#rd_kafka_DescribeConfigs ⇒ Object

DescribeConfigs

# File 'lib/kafka/ffi.rb', line 699

attach_function :rd_kafka_DescribeConfigs, [Client, :pointer, :size_t, Admin::AdminOptions, Queue], :void

#rd_kafka_msg_partitioner_random ⇒ Object

Partitioners

# File 'lib/kafka/ffi.rb', line 598

attach_function :rd_kafka_msg_partitioner_random, [Topic, :string, :size_t, :int32, Opaque, Opaque], :partition

#rd_kafka_ResourceType_name ⇒ Object

Resource Type (enum)

# File 'lib/kafka/ffi.rb', line 707

attach_function :rd_kafka_ResourceType_name, [:resource_type], :string

#rd_kafka_subscribe ⇒ Object

High Level Consumer API

# File 'lib/kafka/ffi.rb', line 551

attach_function :rd_kafka_subscribe, [Consumer, TopicPartitionList.by_ref], :error_code