etcd как БД

Иван Стрелков

@i_strel

Что за зверь такой?

Key-value БД
Strongly consistent
Immutable
Транзакционная
Много паттернов построено на основе неё

Key-Value DB interface

Интерфейс запросов БД

Построен на основе GRPC
И ключ, и значение - массивы байт

      
message RangeRequest {
    bytes key = 1;
    bytes range_end = 2;
    int64 limit = 3;
    int64 revision = 4;
    SortOrder sort_order = 5; // NONE | ASCEND | DESCEND
    SortTarget sort_target = 6; // KEY | VERSION | CREATE | MOD | VALUE
    bool serializable = 7;
    bool keys_only = 8;
    bool count_only = 9;
    int64 min_mod_revision = 10;
    int64 max_mod_revision = 11;
    int64 min_create_revision = 12;
    int64 max_create_revision = 13;
}

Пример 1

Ключи:

aa
ab
abb
abc
b
ba

Запрос: { key: 'ab', range_end: 'b' }

Пример 1

Ключи:

aa
ab
abb
abc
b
ba

Запрос: { key: 'ab', range_end: 'b' }

Пример 2

Ключи:

bx/details/1
sc.sub
sc/support
sc/finance
sca
var

Пример 2

Ключи:

bx/details/1
sc.sub
sc/support
sc/finance
sca
var

Пример 2

Ключи:

bx/details/1
sc.sub
sc/support
sc/finance
sca
var

Запрос: { key: 'sc/', range_end: ? }

Пример 2

Ключи:

bx/details/1
sc.sub
sc/support
sc/finance
sca
var

Запрос: { key: 'sc/', range_end: 'sc0' }

      
message KeyValue {
  bytes key = 1;
  int64 create_revision = 2;
  int64 mod_revision = 3;
  int64 version = 4;
  bytes value = 5;
  int64 lease = 6;
}

      
message KeyValue {
  bytes key = 1;
  int64 create_revision = 2;
  int64 mod_revision = 3;
  int64 version = 4;
  bytes value = 5;
  int64 lease = 6;
}

      
message KeyValue {
    bytes key = 1;
  int64 create_revision = 2;
  int64 mod_revision = 3;
  int64 version = 4;
  bytes value = 5;
  int64 lease = 6;
}

Immutability

OP	rev	sc create_rev	sc mod_rev	sc version
- (beginning)	1	0	0	0
put spam test	2	0	0	0
put sc val1	3	3	3	1
put sc val2	4	3	4	2
del sc	5	0	0	0

Key-Value DB operations

put
del range
get range
watch range
txn

Put

      
        message PutRequest {
            bytes key = 1;
            bytes value = 2;
            int64 lease = 3;
            bool prev_kv = 4;
            bool ignore_value = 5;
            bool ignore_lease = 6;
        }

etcdctl put --write-out=json foo bar
etcdctl put bar baz --prev-kv

Delete

      
        message DeleteRangeRequest {
            bytes key = 1;
            bytes range_end = 2;
            bool prev_kv = 3;
        }

etcdctl del --prefix f
etcdctl del f g # delete every key starting with f etcdctl del foo --prev-kv # return prev value

      
message RangeRequest {
    bytes key = 1;
    bytes range_end = 2;
    int64 limit = 3;
    int64 revision = 4;
    SortOrder sort_order = 5; // NONE | ASCEND | DESCEND
    SortTarget sort_target = 6; // KEY | VERSION | CREATE | MOD | VALUE
    bool serializable = 7;
    bool keys_only = 8;
    bool count_only = 9;
    int64 min_mod_revision = 10;
    int64 max_mod_revision = 11;
    int64 min_create_revision = 12;
    int64 max_create_revision = 13;
}

      
message RangeRequest {
    bytes key = 1;
    bytes range_end = 2;
    int64 limit = 3;
    int64 revision = 4;
    SortOrder sort_order = 5; // NONE | ASCEND | DESCEND
    SortTarget sort_target = 6; // KEY | VERSION | CREATE | MOD | VALUE
    bool serializable = 7;
    bool keys_only = 8;
    bool count_only = 9;
    int64 min_mod_revision = 10;
    int64 max_mod_revision = 11;
    int64 min_create_revision = 12;
    int64 max_create_revision = 13;
}

      
message RangeRequest {
    bytes key = 1;
    bytes range_end = 2;
    int64 limit = 3;
    int64 revision = 4;
    SortOrder sort_order = 5; // NONE | ASCEND | DESCEND
    SortTarget sort_target = 6; // KEY | VERSION | CREATE | MOD | VALUE
    bool serializable = 7;
    bool keys_only = 8;
    bool count_only = 9;
    int64 min_mod_revision = 10;
    int64 max_mod_revision = 11;
    int64 min_create_revision = 12;
    int64 max_create_revision = 13;
}

      
message RangeRequest {
    bytes key = 1;
    bytes range_end = 2;
    int64 limit = 3;
    int64 revision = 4;
    SortOrder sort_order = 5; // NONE | ASCEND | DESCEND
    SortTarget sort_target = 6; // KEY | VERSION | CREATE | MOD | VALUE
    bool serializable = 7;
    bool keys_only = 8;
    bool count_only = 9;
    int64 min_mod_revision = 10;
    int64 max_mod_revision = 11;
    int64 min_create_revision = 12;
    int64 max_create_revision = 13;
}

Get

etcdctl get foo
etcdctl get f g # get range of every key starting with f
etcdctl get --prefix f # the same as the previous
etcdctl get --rev=1 foo # get key from some specific revision

Watch

etcdctl watch foo # watch for key
etcdctl watch --prefix sc/ # watch for every change in the prefix
etcdctl watch f h # watch for range
etcdctl watch --rev=1 --prefix "" # display whole history since the beginning

Transaction

Transaction - атомарная операция в стиле if/then/else

        
  if conditions {
    return success();
  } else {
    return failure();
  }

Transaction - protobuf

      
message TxnRequest {
  repeated Compare compare = 1;
  repeated RequestOp success = 2;
  repeated RequestOp failure = 3;
}

Transaction - вид условия

      
message Compare {
  CompareResult result = 1 // EQUAL | GREATER | LESS | NOT_EQUAL
  CompareTarget target = 2; // VERSION | CREATE | MOD | VALUE

  bytes key = 3;

  oneof target_union {
    int64 version = 4;
    int64 create_revision = 5;
    int64 mod_revision = 6;
    bytes value = 7;
  }
}

Transaction - вид операции

      
message RequestOp {
  oneof request {
    RangeRequest request_range = 1;
    PutRequest request_put = 2;
    DeleteRangeRequest request_delete_range = 3;
  }
}

Transaction - пример

      
$ etcdctl txn --interactive
compares:
mod("some-value") = "0"
mod("different-value") = "0"

success requests (get, put, del):
put some-value a
get --prefix ""
put different-value c
del foo

failure requests (get, put, del):
get some-value

Lease

имеет TTL (но не в том смысле, как мы привыкли)
привязывается к ключу (и отвязывается)
дорогостоящая сущность для etcd
при уничтожении убирает ключи, которые были приаттачены

Lease - Put

      
        message PutRequest {
            bytes key = 1;
            bytes value = 2;
            int64 lease = 3;
            bool prev_kv = 4;
            bool ignore_value = 5;
            bool ignore_lease = 6;
        }

Lease - Put

      
        message PutRequest {
            bytes key = 1;
            bytes value = 2;
            int64 lease = 3;
            bool prev_kv = 4;
            bool ignore_value = 5;
            bool ignore_lease = 6;
        }

Lease

            
$ etcdctl lease grant 60 # lease for 60 seconds
lease 694d74c1abd2ce10 granted with TTL(60s)

$ etcdctl lease keepalive 694d74c1abd2ce10 # prolong

$ etcdctl put --lease=694d74c1abd2ce10 foo bar # put with lease

$ etcdctl put --lease=0 --ignore-value foo # drop lease

$ etcdctl put --lease=694d74c1abd2ce10 --ignore-value foo # attach lease

$ etcdctl lease revoke 694d74c1abd2ce10 # revoke lease

Паттерны etcd

Mutex
RWMutex
Leader election
Queue
и другие...

https://godoc.org/github.com/etcd-io/etcd/contrib/recipes

Mutex

        
mutex := concurrency.NewMutex(s, "seller/235")
if err := mutex.Lock(ctx); err != nil {
  return err
}

doSomeCriticalWork();

if err := mutex.Unlock(context.TODO()); err != nil {
  return err
}

Mutex

      
rwMutex := recipe.NewRWMutex(s, "seller/432")
if err := rwMutex.RLock(ctx); err != nil {
  return err
}

doSomelWorkWhileEntityIsUnchanged();

if err := rwMutex.RUnlock(context.TODO()); err != nil {
  return err
}

Mutex - Writer

      
rwMutex := recipe.NewRWMutex(s, "seller/432")
if err := rwMutex.Lock(ctx); err != nil {
  return err
}

doSomeCriticalWork();

if err := rwMutex.Unlock(context.TODO()); err != nil {
  return err
}

Leader election

      
e := concurrency.NewElection(s, "my-db-leader")

if err := e.Campaign(ctx, myProposal); err != nil {
  log.Fatal(err)
}

doSomethingAsALeader()

if err := e.Resign(ctx); err != nil {
  log.Fatal(err)
}

Consistency

Strict serializable

RAFT

Постоянно пингует на предмет network partition
Каждая запись фиксируется только после того, как подтвердится большинством нод
Кластер становится недоступным в случае network partition (даже на чтение!)

Из нерассказанного...

Управление доступом
Внутреннее устройство
Перфоманс

Спасибо за внимание!

Иван Стрелков, OZON

Презентация: istrel.github.io/etcd/