diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..30ad53c77 --- /dev/null +++ b/Makefile @@ -0,0 +1,16 @@ +# Determine this makefile's path. +# Be sure to place this BEFORE `include` directives, if any. +THIS_FILE := $(lastword $(MAKEFILE_LIST)) + + +### oplog requires protoc-gen-go v1.20.0 or later +# GO111MODULE=on go get -u google.golang.org/protobuf/cmd/protoc-gen-go@v1.20.1 +proto: + protoc --go_out=paths=source_relative:. ./internal/oplog/any_operation.proto + protoc --go_out=paths=source_relative:. ./internal/oplog/store/oplog.proto + protoc-go-inject-tag -input=./internal/oplog/store/oplog.pb.go + protoc --go_out=paths=source_relative:. ./internal/oplog/oplog_test/oplog_test.proto + protoc-go-inject-tag -input=./internal/oplog/oplog_test/oplog_test.pb.go + + +.PHONY: proto \ No newline at end of file diff --git a/go.mod b/go.mod index d00b37c33..c3e7c54db 100644 --- a/go.mod +++ b/go.mod @@ -6,15 +6,19 @@ require ( github.com/armon/go-metrics v0.3.3 github.com/fatih/color v1.9.0 github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 + github.com/golang-migrate/migrate/v4 v4.9.1 + github.com/golang/protobuf v1.4.0-rc.4 github.com/hashicorp/errwrap v1.0.0 github.com/hashicorp/go-alpnmux v0.0.0-20200323180452-dee08f00df54 github.com/hashicorp/go-hclog v0.12.1 github.com/hashicorp/go-kms-wrapping v0.5.7 github.com/hashicorp/go-multierror v1.0.0 + github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/vault v1.2.1-0.20200310200732-a321b1217d5a github.com/hashicorp/vault/api v1.0.5-0.20200310200732-a321b1217d5a github.com/hashicorp/vault/sdk v0.1.14-0.20200310200732-a321b1217d5a + github.com/jinzhu/gorm v1.9.12 github.com/kr/text v0.2.0 github.com/lib/pq v1.3.0 github.com/mattn/go-colorable v0.1.6 @@ -29,4 +33,5 @@ require ( golang.org/x/net v0.0.0-20200301022130-244492dfa37a google.golang.org/grpc v1.27.1 google.golang.org/protobuf v1.20.1 + gotest.tools v2.2.0+incompatible ) diff --git a/internal/oplog/README.md b/internal/oplog/README.md new file mode 100644 index 000000000..eac3e4d7d --- /dev/null +++ b/internal/oplog/README.md @@ -0,0 +1,174 @@ +# oplog + +oplog is a package for writing operation log (oplog) entries for the purpose of replication and verification of the data stored in the Watchtower RDBMS. + +- [oplog](#oplog) + - [Usage](#usage) + - [TBD/TODO](#tbdtodo) + - [oplog entry](#oplog-entry) + - [oplog tables](#oplog-tables) + - [oplog optimistic locking using tickets](#oplog-optimistic-locking-using-tickets) +## Usage +```go + +// you must init the ticket in its own transaction. You only need +// to init a ticket once in the database. It doesn't need to happen for +// every connection. Once it's persistent in the database, you can simply Get it. +initTx := db.Begin() +ticketer := &GormTicketer{Tx: initTx} +err = ticketer.InitTicket("users") +// if there's no error, then commit the initialized ticket +initTx.Commit() + +userCreate := oplog_test.TestUser{ + TestUser: oplog_test.TestUser{ + Name: userName, + }, +} +tx := db.Begin() +// write the user to the database +tx.Create(&userCreate) + +ticketer = &GormTicketer{Tx: db} + +// get a ticket for writing users to the oplog +ticket, err := ticketer.GetTicket("users") + +// create an entry for the oplog with the entry's metadata (likely the entry's Scope) +newLogEntry := NewEntry( + "test-users", + []Metadata{ + Metadata{ + Key: "deployment", + Value: "amex", + }, + Metadata{ + Key: "project", + Value: "central-info-systems", + }, + }, + cipherer, // wrapping.Wrapper + ticketer, +) + +// write an entry with N messages (variadic parameter) in the order they were sent to the database +_, err = newLogEntry.WriteEntryWith( + context.Background(), + &GormWriter{tx}, + ticket, + &Message{Message: &userCreate, TypeName: "user", OpType: OpType_CREATE_OP}, +) +// if there's an error writing the oplog then roll EVERYTHING back +if err != nil { + tx.Rollback() +} +// no err means you can commit all the things. +tx.Commit() +``` +## TBD/TODO +We need to discuss and decide how Watchtower is going to handle the following oplog things: + +* SQL migrations: you'll find the package's [SQL migrations](https://github.com/golang-migrate/migrate) under: ./migrations/postgres We need to decide how Watchtower will manage migrations across the system and we will likely need to reference this package's migrations somehow. + +## oplog entry +``` + Example Oplog Entry for the Target Aggregate +┌────────────────────────────────────────────────┐ +│ FIFO with []byte buffer │ +│ │ +│┌─Msg 4────┐┌─Msg 3────┐┌─Msg 2────┐ │ +││ ││ ││ │ │ +││ Tags ││ Host ││ HostSet │ │ +││┌────────┐││┌────────┐││┌────────┐│ │ +│││ ││││ ││││ ││ │ +│││ ││││ ││││ ││ │ +│││ Tag ││││ Host ││││HostSet ││ │ +│││protobuf││││protobuf││││protobuf││ │ +│││ ││││ ││││ ││ │ +│││ ││││ ││││ ││ │ +││└────────┘││└────────┘││└────────┘│ │ +││┌────────┐││┌────────┐││┌────────┐│ │ +│││ ││││ ││││ ││ │ +│││typeName││││typeName││││typeName││ │ +│││ Tag ││││ Host ││││HostSet ││ │ +││└────────┘││└────────┘││└────────┘│ │ +││┌────────┐││┌────────┐││┌────────┐│ │ +│││ ││││ ││││ ││ │ +│││ OpType ││││ OpType ││││ OpType ││ │ +│││ Create ││││ Create ││││ Create ││ │ +││└────────┘││└────────┘││└────────┘│ │ +│└──────────┘└──────────┘└──────────┘ │ +└────────────────────────────────────────────────┘ +``` + +## oplog tables + +``` + oplog tables: + as the diagram shows, we can split the + oplog_entries into multiple tables if + needed for performance. + +┌────────────────┐ +│┌───────────────┴┐ +││┌───────────────┴┐ ┌────────────────┐ +│││ oplog_entries │ │ oplog_ticket │ +││├────────────────┤ ├────────────────┤ +│││id │╲ │id │ +│││aggregate_name │──┼───────┼ │aggregate_name │ +└┤│data │╱ │version │ + └┤ │ │ │ + └────────────────┘ └────────────────┘ + ┼ + │ + │ + ┼ + ╱│╲ + ┌────────────────┐ + │ oplog_metadata │ + ├────────────────┤ + │id │ + │entry_id │ + │key │ + │value │ + └────────────────┘ + ``` + +## oplog optimistic locking using tickets + ``` + Alice's Database + transaction Bob's transaction + + │ │ │ + │─────────BEGIN──────────────────▶│ │ + │ │◀───────────────BEGIN─────────────────┤ + │ ┌───────────┴───────────┐ │ + │ │ ticket version:1 │ │ + │ └───────────┬───────────┘ │ + │ Select oplog-ticket for │ │ + ├──────────────"Target"──────────▶│ Select oplog-ticket for │ + │ │◀──────────────"Target"───────────────┤ + │ │ │ + │ Write to Tables in │ │ + ├───────Target Aggregate─────────▶│ │ + │ │ Write to Tables in │ + │ │◀─────────Target Aggregate────────────┤ + │ │ │ + │ │ │ + │ Update Ticket │ │ + ├────Version = 2 where───────────▶│ │ + │ Version = 1 ┌─────────┴─────────┐ │ + │ │ ticket version: 2 │ │ + │ └─────────┬─────────┘ │ + │ │ Update Ticket │ + │ │◀───────────Version = 2 where─────────┤ + │ │ Version = 1 ┌────┴──────────┐ + │ │ │ update failed │ + ├────────────Commit──────────────▶│ └────┬──────────┘ + │ │ │ + │ │ │ + │ │ │ + │ │◀────────────Rollback─────────────────┤ + │ │ │ + │ │ │ + ``` diff --git a/internal/oplog/any_operation.pb.go b/internal/oplog/any_operation.pb.go new file mode 100644 index 000000000..0e2112c93 --- /dev/null +++ b/internal/oplog/any_operation.pb.go @@ -0,0 +1,263 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.20.1 +// protoc v3.11.4 +// source: internal/oplog/any_operation.proto + +package oplog + +import ( + proto "github.com/golang/protobuf/proto" + field_mask "google.golang.org/genproto/protobuf/field_mask" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +// OpType provides the type of operation the Any message represents (create, +// update, delete) +type OpType int32 + +const ( + // UNKNOWN_OP is an unknown operation + OpType_UNKNOWN_OP OpType = 0 + // CREATE_OP is a create operation + OpType_CREATE_OP OpType = 1 + // UPDATE_OP is an update operation + OpType_UPDATE_OP OpType = 2 + // DELETE_OP is a delete operation + OpType_DELETE_OP OpType = 3 +) + +// Enum value maps for OpType. +var ( + OpType_name = map[int32]string{ + 0: "UNKNOWN_OP", + 1: "CREATE_OP", + 2: "UPDATE_OP", + 3: "DELETE_OP", + } + OpType_value = map[string]int32{ + "UNKNOWN_OP": 0, + "CREATE_OP": 1, + "UPDATE_OP": 2, + "DELETE_OP": 3, + } +) + +func (x OpType) Enum() *OpType { + p := new(OpType) + *p = x + return p +} + +func (x OpType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (OpType) Descriptor() protoreflect.EnumDescriptor { + return file_internal_oplog_any_operation_proto_enumTypes[0].Descriptor() +} + +func (OpType) Type() protoreflect.EnumType { + return &file_internal_oplog_any_operation_proto_enumTypes[0] +} + +func (x OpType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use OpType.Descriptor instead. +func (OpType) EnumDescriptor() ([]byte, []int) { + return file_internal_oplog_any_operation_proto_rawDescGZIP(), []int{0} +} + +// AnyOperation provides a message for anything and the type of operation it +// represents. +type AnyOperation struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TypeName string `protobuf:"bytes,1,opt,name=type_name,json=typeName,proto3" json:"type_name,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + OperationType OpType `protobuf:"varint,3,opt,name=operation_type,json=operationType,proto3,enum=hashicorp.watchtower.controller.oplog.v1.OpType" json:"operation_type,omitempty"` + // mask is the mask of fields to update. + FieldMask *field_mask.FieldMask `protobuf:"bytes,4,opt,name=field_mask,json=fieldMask,proto3" json:"field_mask,omitempty"` +} + +func (x *AnyOperation) Reset() { + *x = AnyOperation{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_oplog_any_operation_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AnyOperation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AnyOperation) ProtoMessage() {} + +func (x *AnyOperation) ProtoReflect() protoreflect.Message { + mi := &file_internal_oplog_any_operation_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AnyOperation.ProtoReflect.Descriptor instead. +func (*AnyOperation) Descriptor() ([]byte, []int) { + return file_internal_oplog_any_operation_proto_rawDescGZIP(), []int{0} +} + +func (x *AnyOperation) GetTypeName() string { + if x != nil { + return x.TypeName + } + return "" +} + +func (x *AnyOperation) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *AnyOperation) GetOperationType() OpType { + if x != nil { + return x.OperationType + } + return OpType_UNKNOWN_OP +} + +func (x *AnyOperation) GetFieldMask() *field_mask.FieldMask { + if x != nil { + return x.FieldMask + } + return nil +} + +var File_internal_oplog_any_operation_proto protoreflect.FileDescriptor + +var file_internal_oplog_any_operation_proto_rawDesc = []byte{ + 0x0a, 0x22, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6f, 0x70, 0x6c, 0x6f, 0x67, + 0x2f, 0x61, 0x6e, 0x79, 0x5f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x28, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x77, 0x61, 0x74, 0x63, 0x68, 0x74, 0x6f, 0x77, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x6f, 0x70, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x1a, 0x20, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x22, 0xd5, 0x01, 0x0a, 0x0c, 0x41, 0x6e, 0x79, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x12, 0x57, 0x0a, 0x0e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x30, 0x2e, 0x68, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x77, 0x61, 0x74, 0x63, 0x68, 0x74, 0x6f, + 0x77, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x6f, + 0x70, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0d, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x39, 0x0a, + 0x0a, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x73, 0x6b, 0x52, 0x09, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x73, 0x6b, 0x2a, 0x45, 0x0a, 0x06, 0x4f, 0x70, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x4f, 0x50, + 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x5f, 0x4f, 0x50, 0x10, + 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x4f, 0x50, 0x10, 0x02, + 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x4f, 0x50, 0x10, 0x03, 0x42, + 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x77, 0x61, 0x74, 0x63, 0x68, 0x74, 0x6f, 0x77, + 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6f, 0x70, 0x6c, 0x6f, + 0x67, 0x2f, 0x6f, 0x70, 0x6c, 0x6f, 0x67, 0x3b, 0x6f, 0x70, 0x6c, 0x6f, 0x67, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_internal_oplog_any_operation_proto_rawDescOnce sync.Once + file_internal_oplog_any_operation_proto_rawDescData = file_internal_oplog_any_operation_proto_rawDesc +) + +func file_internal_oplog_any_operation_proto_rawDescGZIP() []byte { + file_internal_oplog_any_operation_proto_rawDescOnce.Do(func() { + file_internal_oplog_any_operation_proto_rawDescData = protoimpl.X.CompressGZIP(file_internal_oplog_any_operation_proto_rawDescData) + }) + return file_internal_oplog_any_operation_proto_rawDescData +} + +var file_internal_oplog_any_operation_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_internal_oplog_any_operation_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_internal_oplog_any_operation_proto_goTypes = []interface{}{ + (OpType)(0), // 0: hashicorp.watchtower.controller.oplog.v1.OpType + (*AnyOperation)(nil), // 1: hashicorp.watchtower.controller.oplog.v1.AnyOperation + (*field_mask.FieldMask)(nil), // 2: google.protobuf.FieldMask +} +var file_internal_oplog_any_operation_proto_depIdxs = []int32{ + 0, // 0: hashicorp.watchtower.controller.oplog.v1.AnyOperation.operation_type:type_name -> hashicorp.watchtower.controller.oplog.v1.OpType + 2, // 1: hashicorp.watchtower.controller.oplog.v1.AnyOperation.field_mask:type_name -> google.protobuf.FieldMask + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_internal_oplog_any_operation_proto_init() } +func file_internal_oplog_any_operation_proto_init() { + if File_internal_oplog_any_operation_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_internal_oplog_any_operation_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AnyOperation); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_internal_oplog_any_operation_proto_rawDesc, + NumEnums: 1, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_internal_oplog_any_operation_proto_goTypes, + DependencyIndexes: file_internal_oplog_any_operation_proto_depIdxs, + EnumInfos: file_internal_oplog_any_operation_proto_enumTypes, + MessageInfos: file_internal_oplog_any_operation_proto_msgTypes, + }.Build() + File_internal_oplog_any_operation_proto = out.File + file_internal_oplog_any_operation_proto_rawDesc = nil + file_internal_oplog_any_operation_proto_goTypes = nil + file_internal_oplog_any_operation_proto_depIdxs = nil +} diff --git a/internal/oplog/any_operation.proto b/internal/oplog/any_operation.proto new file mode 100644 index 000000000..81abc4e2c --- /dev/null +++ b/internal/oplog/any_operation.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +import "google/protobuf/field_mask.proto"; + +package hashicorp.watchtower.controller.oplog.v1; +option go_package = "github.com/hashicorp/watchtower/internal/oplog/oplog;oplog"; + +// OpType provides the type of operation the Any message represents (create, +// update, delete) +enum OpType { + // UNKNOWN_OP is an unknown operation + UNKNOWN_OP = 0; + // CREATE_OP is a create operation + CREATE_OP = 1; + // UPDATE_OP is an update operation + UPDATE_OP = 2; + // DELETE_OP is a delete operation + DELETE_OP = 3; +} + +// AnyOperation provides a message for anything and the type of operation it +// represents. +message AnyOperation { + string type_name = 1; + bytes value = 2; + OpType operation_type = 3; + // mask is the mask of fields to update. + google.protobuf.FieldMask field_mask = 4; +} \ No newline at end of file diff --git a/internal/oplog/docs.go b/internal/oplog/docs.go new file mode 100644 index 000000000..fab0a3b9b --- /dev/null +++ b/internal/oplog/docs.go @@ -0,0 +1,3 @@ +// oplog is a package for writing operational log (oplog) entries for the purpose of replication and verification of the data stored in the Watchtower RDBMS. +// please see README.md in this directory for more information. +package oplog diff --git a/internal/oplog/migrations/postgres/01_db.down.sql b/internal/oplog/migrations/postgres/01_db.down.sql new file mode 100644 index 000000000..6420e36ac --- /dev/null +++ b/internal/oplog/migrations/postgres/01_db.down.sql @@ -0,0 +1,5 @@ +drop table if exists oplog_entry; +drop table if exists oplog_metadata; +drop table if exists oplog_ticket; +drop index if exists idx_oplog_metatadata_key; +drop index if exists idx_oplog_metatadata_value \ No newline at end of file diff --git a/internal/oplog/migrations/postgres/01_db.up.sql b/internal/oplog/migrations/postgres/01_db.up.sql new file mode 100644 index 000000000..b9d478142 --- /dev/null +++ b/internal/oplog/migrations/postgres/01_db.up.sql @@ -0,0 +1,24 @@ +CREATE TABLE if not exists oplog_entry ( + id bigint generated always as identity primary key, + create_time timestamp with time zone default current_timestamp, + update_time timestamp with time zone default current_timestamp, + version text NOT NULL, + aggregate_name text NOT NULL, + "data" bytea NOT NULL +); +CREATE TABLE if not exists oplog_ticket ( + id bigint generated always as identity primary key, + create_time timestamp with time zone default current_timestamp, + update_time timestamp with time zone default current_timestamp, + "name" text NOT NULL UNIQUE, + "version" bigint NOT NULL +); +CREATE TABLE if not exists oplog_metadata ( + id bigint generated always as identity primary key, + create_time timestamp with time zone default current_timestamp, + entry_id bigint NOT NULL REFERENCES oplog_entry(id) ON DELETE CASCADE ON UPDATE CASCADE, + "key" text NOT NULL, + value text NULL +); +create index if not exists idx_oplog_metatadata_key on oplog_metadata(key); +create index if not exists idx_oplog_metatadata_value on oplog_metadata(value); \ No newline at end of file diff --git a/internal/oplog/oplog.go b/internal/oplog/oplog.go new file mode 100644 index 000000000..a2dd95de7 --- /dev/null +++ b/internal/oplog/oplog.go @@ -0,0 +1,237 @@ +package oplog + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "io" + + protoV1 "github.com/golang/protobuf/proto" + wrapping "github.com/hashicorp/go-kms-wrapping" + "github.com/hashicorp/watchtower/internal/oplog/store" + _ "github.com/lib/pq" + "google.golang.org/protobuf/proto" +) + +// Version of oplog entries (among other things, it's used to manage upgrade compatibility when replicating) +const Version = "v1" + +// Message wraps a proto.Message and adds a operation type (Create, Update, Delete) +type Message struct { + proto.Message + TypeName string + OpType OpType + FieldMaskPaths []string +} + +// Entry represents an oplog entry +type Entry struct { + *store.Entry + Cipherer wrapping.Wrapper `sql:"-"` + Ticketer Ticketer `sql:"-"` +} + +// Metadata provides meta information about the Entry +type Metadata map[string][]string + +// NewEntry creates a new Entry +func NewEntry(aggregateName string, metadata Metadata, cipherer wrapping.Wrapper, ticketer Ticketer) (*Entry, error) { + entry := Entry{ + Entry: &store.Entry{ + AggregateName: aggregateName, + Version: Version, + }, + Cipherer: cipherer, + Ticketer: ticketer, + } + if len(metadata) > 0 { + storeMD := []*store.Metadata{} + for k, v := range metadata { + if len(v) > 0 { + for _, vv := range v { + storeMD = append(storeMD, &store.Metadata{Key: k, Value: vv}) + } + continue + } + // this metadata just has a key with no values + storeMD = append(storeMD, &store.Metadata{Key: k}) + } + entry.Metadata = storeMD + } + if err := entry.validate(); err != nil { + return nil, fmt.Errorf("error creating entry: %w", err) + } + return &entry, nil +} +func (e *Entry) validate() error { + if e.Cipherer == nil { + return errors.New("entry Cipherer is nil") + } + if e.Ticketer == nil { + return errors.New("entry Ticketer is nil") + } + if e.Entry == nil { + return errors.New("store.Entry is nil") + } + if e.Entry.Version == "" { + return errors.New("entry version is not set") + } + if e.Entry.AggregateName == "" { + return errors.New("entry aggregate name is not set") + } + return nil +} + +// UnmarshalData the data attribute from []byte (treated as a FIFO QueueBuffer) to a []proto.Message +func (e *Entry) UnmarshalData(types *TypeCatalog) ([]Message, error) { + if types == nil { + return nil, errors.New("TypeCatalog is nil") + } + if len(e.Data) == 0 { + return nil, errors.New("no Data to unmarshal") + } + msgs := []Message{} + queue := Queue{ + Buffer: *bytes.NewBuffer(e.Data), + Catalog: types, + } + for { + m, typ, fieldMaskPaths, err := queue.Remove() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("error removing item from queue: %w", err) + } + name, err := types.GetTypeName(m) + if err != nil { + return nil, fmt.Errorf("error getting TypeName: %w", err) + } + msgs = append(msgs, Message{Message: m, TypeName: name, OpType: typ, FieldMaskPaths: fieldMaskPaths}) + } + return msgs, nil +} + +// WriteEntryWith the []proto.Message marshaled into the entry data as a FIFO QueueBuffer +// if Cipherer != nil then the data is authentication encrypted +func (e *Entry) WriteEntryWith(ctx context.Context, tx Writer, ticket *store.Ticket, msgs ...*Message) error { + if tx == nil { + return errors.New("bad writer") + } + if err := e.validate(); err != nil { + return fmt.Errorf("error vetting entry for writing: %w", err) + } + if ticket == nil || ticket.Version == 0 { + return errors.New("bad ticket") + } + queue := Queue{} + for _, m := range msgs { + if m == nil { + return errors.New("bad message") + } + if err := queue.Add(m.Message, m.TypeName, m.OpType, WithFieldMaskPaths(m.FieldMaskPaths)); err != nil { + return fmt.Errorf("error adding message to queue: %w", err) + } + } + e.Data = append(e.Data, []byte(queue.Bytes())...) + + if e.Cipherer != nil { + if err := e.EncryptData(ctx); err != nil { + return fmt.Errorf("error encrypting entry: %w", err) + } + } + if err := tx.Create(e); err != nil { + return fmt.Errorf("error writing data to storage: %w", err) + } + return e.Ticketer.Redeem(ticket) +} + +// Write the entry as is with whatever it has for e.Data marshaled into a FIFO QueueBuffer +// Cipherer != nil then the data is authentication encrypted +func (e *Entry) Write(ctx context.Context, tx Writer, ticket *store.Ticket) error { + if err := e.validate(); err != nil { + return fmt.Errorf("error vetting entry for writing: %w", err) + } + if ticket == nil || ticket.Version == 0 { + return errors.New("bad ticket") + } + if e.Cipherer != nil { + if err := e.EncryptData(ctx); err != nil { + return fmt.Errorf("error encrypting entry: %w", err) + } + } + if err := tx.Create(e); err != nil { + return fmt.Errorf("error writing data to storage: %w", err) + } + return e.Ticketer.Redeem(ticket) +} + +// EncryptData the entry's data using its Cipherer (wrapping.Wrapper) +func (e *Entry) EncryptData(ctx context.Context) error { + d, err := e.Cipherer.Encrypt(ctx, e.Data, nil) + if err != nil { + return fmt.Errorf("error encrypting entry: %w", err) + } + blob, err := protoV1.Marshal(d) + if err != nil { + return fmt.Errorf("error marshaling encrypted data: %w", err) + } + e.Data = []byte(base64.RawURLEncoding.EncodeToString(blob)) + return nil +} + +// DecryptData will decrypt the entry's data using its Cipherer (wrapping.Wrapper) +func (e *Entry) DecryptData(ctx context.Context) error { + blob, err := base64.RawURLEncoding.DecodeString(string(e.Data)) + if err != nil { + return fmt.Errorf("error decoding encrypted data: %w", err) + } + var msg wrapping.EncryptedBlobInfo + err = protoV1.Unmarshal(blob, &msg) + if err != nil { + return fmt.Errorf("error unmarshaling encrypted data: %w", err) + } + d, err := e.Cipherer.Decrypt(ctx, &msg, nil) + if err != nil { + return fmt.Errorf("error decrypting data: %w", err) + } + e.Data = d + return nil +} + +// Replay provides the ability to replay an entry. you must initialize any new tables ending with the tableSuffix before +// calling Replay, otherwise you'll get "a table doesn't exist" error. +func (e *Entry) Replay(ctx context.Context, tx Writer, types *TypeCatalog, tableSuffix string) error { + msgs, err := e.UnmarshalData(types) + if err != nil { + return fmt.Errorf("error on UnmarshalData: %w", err) + } + for _, m := range msgs { + em, ok := m.Message.(ReplayableMessage) + if !ok { + return fmt.Errorf("%T is not a ReplayableMessage", m.Message) + } + origTableName := em.TableName() + defer em.SetTableName(origTableName) + em.SetTableName(origTableName + tableSuffix) + switch m.OpType { + case OpType_CREATE_OP: + if err := tx.Create(m.Message); err != nil { + return fmt.Errorf("replay error: %w", err) + } + case OpType_UPDATE_OP: + if err := tx.Update(m.Message, m.FieldMaskPaths); err != nil { + return fmt.Errorf("replay error: %w", err) + } + case OpType_DELETE_OP: + if err := tx.Delete(m.Message); err != nil { + return fmt.Errorf("replay error: %w", err) + } + default: + return fmt.Errorf("replay error: unhandled operation %T", m.OpType) + } + } + return nil +} diff --git a/internal/oplog/oplog_test.go b/internal/oplog/oplog_test.go new file mode 100644 index 000000000..4bc157769 --- /dev/null +++ b/internal/oplog/oplog_test.go @@ -0,0 +1,886 @@ +package oplog + +import ( + "context" + "crypto/rand" + "database/sql" + "fmt" + "os" + "strings" + "sync" + "testing" + + wrapping "github.com/hashicorp/go-kms-wrapping" + "github.com/hashicorp/go-kms-wrapping/wrappers/aead" + "github.com/hashicorp/go-uuid" + "github.com/hashicorp/watchtower/internal/oplog/oplog_test" + "github.com/jinzhu/gorm" + "gotest.tools/assert" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/postgres" + _ "github.com/golang-migrate/migrate/v4/source/file" + "github.com/ory/dockertest/v3" +) + +// Need a way to manage the shared database resource for these oplog +// tests, so runningTests gives us a waitgroup to do this and clean up +// the shared database resource when we're done. +var runningTests sync.WaitGroup + +// startTest signals that we're starting a test that uses the shared test resources +func startTest() { + runningTests.Add(1) +} + +// completeTest signals that we've finished a test that uses the shared test resources +func completeTest() { + runningTests.Done() +} + +// waitForTests will wait for all the tests that are sharing resources like the database +func waitForTests() { + runningTests.Wait() +} + +// testDatabaseURL is initialized once using sync.Once and set to the database URL for testing +var testDatabaseURL string + +// testInitDatabase ensures that the database is only initialized once during the tests. +var testInitDatabase sync.Once + +// setup the tests (initialize the database one-time and intialized testDatabaseURL) +func setup(t *testing.T) (func(), string) { + cleanup := func() {} + var url string + var err error + testInitDatabase.Do(func() { + cleanup, url, err = initDbInDocker(t) + if err != nil { + panic(err) + } + testDatabaseURL = url + db, err := test_dbconn(url) + if err != nil { + panic(err) + } + defer db.Close() + oplog_test.Init(db) + }) + return cleanup, testDatabaseURL +} + +// Test_BasicOplog provides some basic unit tests for oplogs +func Test_BasicOplog(t *testing.T) { + t.Parallel() + startTest() + cleanup, url := setup(t) + defer cleanup() + defer completeTest() // must come after the "defer cleanup()" + db, err := test_dbconn(url) + assert.NilError(t, err) + defer db.Close() + + t.Run("EncryptData/DecryptData/UnmarshalData", func(t *testing.T) { + cipherer := initWrapper(t) + + // now let's us optimistic locking via a ticketing system for a serialized oplog + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + assert.NilError(t, err) + + types, err := NewTypeCatalog(Type{new(oplog_test.TestUser), "user"}) + assert.NilError(t, err) + queue := Queue{Catalog: types} + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + + user := oplog_test.TestUser{ + Name: "foo-" + id, + } + + resp := db.Create(&user) + assert.NilError(t, resp.Error) + + err = queue.Add(&user, "user", OpType_CREATE_OP) + assert.NilError(t, err) + l, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + l.Data = queue.Bytes() + + err = l.EncryptData(context.Background()) + assert.NilError(t, err) + + resp = db.Create(&l) + assert.NilError(t, resp.Error) + entryId := l.Id + + var foundEntry Entry + err = db.Where("id = ?", entryId).First(&foundEntry).Error + assert.NilError(t, err) + foundEntry.Cipherer = cipherer + err = foundEntry.DecryptData(context.Background()) + assert.NilError(t, err) + + foundUsers, err := foundEntry.UnmarshalData(types) + assert.NilError(t, err) + assert.Assert(t, foundUsers[0].Message.(*oplog_test.TestUser).Name == user.Name) + foundUsers, err = foundEntry.UnmarshalData(types) + assert.NilError(t, err) + assert.Assert(t, foundUsers[0].Message.(*oplog_test.TestUser).Name == user.Name) + }) + + t.Run("write entry", func(t *testing.T) { + cipherer := initWrapper(t) + + // now let's us optimistic locking via a ticketing system for a serialized oplog + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + assert.NilError(t, err) + + types, err := NewTypeCatalog(Type{new(oplog_test.TestUser), "user"}) + assert.NilError(t, err) + + queue := Queue{Catalog: types} + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + + user := oplog_test.TestUser{ + Name: "foo-" + id, + } + + resp := db.Create(&user) + assert.NilError(t, resp.Error) + + ticketName, err := uuid.GenerateUUID() + assert.NilError(t, err) + err = ticketer.InitTicket(ticketName) + assert.NilError(t, err) + ticket, err := ticketer.GetTicket(ticketName) + assert.NilError(t, err) + + queue = Queue{} + err = queue.Add(&user, "user", OpType_CREATE_OP) + assert.NilError(t, err) + + newLogEntry, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + newLogEntry.Data = queue.Bytes() + err = newLogEntry.Write(context.Background(), &GormWriter{db}, ticket) + assert.NilError(t, err) + assert.Assert(t, newLogEntry.Id != 0) + }) + +} + +// Test_NewEntry provides some basic unit tests for NewEntry +func Test_NewEntry(t *testing.T) { + t.Parallel() + startTest() + cleanup, url := setup(t) + defer cleanup() + defer completeTest() // must come after the "defer cleanup()" + db, err := test_dbconn(url) + assert.NilError(t, err) + defer db.Close() + t.Run("valid", func(t *testing.T) { + cipherer := initWrapper(t) + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + assert.NilError(t, err) + + _, err = NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + }) + t.Run("no metadata success", func(t *testing.T) { + cipherer := initWrapper(t) + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + assert.NilError(t, err) + + _, err = NewEntry( + "test-users", + nil, + cipherer, + ticketer, + ) + assert.NilError(t, err) + }) + t.Run("no aggregateName", func(t *testing.T) { + cipherer := initWrapper(t) + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + assert.NilError(t, err) + + _, err = NewEntry( + "", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.Equal(t, err.Error(), "error creating entry: entry aggregate name is not set") + }) + t.Run("bad cipherer", func(t *testing.T) { + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + assert.NilError(t, err) + + _, err = NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + nil, + ticketer, + ) + assert.Equal(t, err.Error(), "error creating entry: entry Cipherer is nil") + }) + t.Run("bad ticket", func(t *testing.T) { + cipherer := initWrapper(t) + _, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + nil, + ) + assert.Equal(t, err.Error(), "error creating entry: entry Ticketer is nil") + }) +} + +func Test_UnmarshalData(t *testing.T) { + t.Parallel() + startTest() + cleanup, url := setup(t) + defer cleanup() + defer completeTest() // must come after the "defer cleanup()" + db, err := test_dbconn(url) + assert.NilError(t, err) + defer db.Close() + + cipherer := initWrapper(t) + + // now let's us optimistic locking via a ticketing system for a serialized oplog + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + + types, err := NewTypeCatalog(Type{new(oplog_test.TestUser), "user"}) + assert.NilError(t, err) + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + + t.Run("valid", func(t *testing.T) { + queue := Queue{Catalog: types} + + user := oplog_test.TestUser{ + Name: "foo-" + id, + } + err = queue.Add(&user, "user", OpType_CREATE_OP) + assert.NilError(t, err) + entry, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + entry.Data = queue.Bytes() + marshaledUsers, err := entry.UnmarshalData(types) + assert.NilError(t, err) + assert.Assert(t, marshaledUsers[0].Message.(*oplog_test.TestUser).Name == user.Name) + + take2marshaledUsers, err := entry.UnmarshalData(types) + assert.NilError(t, err) + assert.Assert(t, take2marshaledUsers[0].Message.(*oplog_test.TestUser).Name == user.Name) + }) + + t.Run("no data", func(t *testing.T) { + queue := Queue{Catalog: types} + entry, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + entry.Data = queue.Bytes() + _, err = entry.UnmarshalData(types) + assert.Equal(t, err.Error(), "no Data to unmarshal") + }) + + t.Run("nil types", func(t *testing.T) { + queue := Queue{Catalog: types} + + user := oplog_test.TestUser{ + Name: "foo-" + id, + } + err = queue.Add(&user, "user", OpType_CREATE_OP) + assert.NilError(t, err) + entry, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + _, err = entry.UnmarshalData(nil) + assert.Equal(t, err.Error(), "TypeCatalog is nil") + }) + + t.Run("missing type", func(t *testing.T) { + queue := Queue{Catalog: types} + + user := oplog_test.TestUser{ + Name: "foo-" + id, + } + err = queue.Add(&user, "user", OpType_CREATE_OP) + assert.NilError(t, err) + entry, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + entry.Data = queue.Bytes() + types, err := NewTypeCatalog(Type{new(oplog_test.TestUser), "not-valid-name"}) + assert.NilError(t, err) + _, err = entry.UnmarshalData(types) + assert.Equal(t, err.Error(), "error removing item from queue: error getting the TypeName for Remove: error typeName is not found for Get") + }) +} + +// Test_Replay provides some basic unit tests for replaying entries +func Test_Replay(t *testing.T) { + startTest() + t.Parallel() + cleanup, url := setup(t) + defer cleanup() + defer completeTest() // must come after the "defer cleanup()" + db, err := test_dbconn(url) + assert.NilError(t, err) + defer db.Close() + + cipherer := initWrapper(t) + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + // setup new tables for replay + tableSuffix := "_" + id + tmpUserModel := &oplog_test.ReplayableTestUser{} + tmpUserModel.SetTableName(fmt.Sprintf("%s%s", tmpUserModel.TableName(), tableSuffix)) + db.AutoMigrate(tmpUserModel) + defer db.DropTableIfExists(tmpUserModel) + + ticketName, err := uuid.GenerateUUID() + assert.NilError(t, err) + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + assert.NilError(t, err) + + err = ticketer.InitTicket(ticketName) + assert.NilError(t, err) + + t.Run("replay:create/update", func(t *testing.T) { + + ticket, err := ticketer.GetTicket(ticketName) + assert.NilError(t, err) + + newLogEntry, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + + tx := db.Begin() + + id3, err := uuid.GenerateUUID() + assert.NilError(t, err) + userName := "foo-" + id3 + // create a user that's replayable + userCreate := oplog_test.ReplayableTestUser{ + TestUser: oplog_test.TestUser{ + Name: userName, + }, + } + err = tx.Create(&userCreate).Error + assert.NilError(t, err) + userSave := oplog_test.ReplayableTestUser{ + TestUser: oplog_test.TestUser{ + Id: userCreate.Id, + Name: userCreate.Name, + Email: userName + "@hashicorp.com", + }, + } + err = tx.Save(&userSave).Error + assert.NilError(t, err) + + userUpdate := oplog_test.ReplayableTestUser{ + TestUser: oplog_test.TestUser{ + Id: userCreate.Id, + PhoneNumber: "867-5309", + }, + } + err = tx.Model(&userUpdate).Updates(map[string]interface{}{"PhoneNumber": "867-5309"}).Error + assert.NilError(t, err) + + err = newLogEntry.WriteEntryWith(context.Background(), &GormWriter{tx}, ticket, + &Message{Message: &userCreate, TypeName: "user", OpType: OpType_CREATE_OP}, + &Message{Message: &userSave, TypeName: "user", OpType: OpType_UPDATE_OP}, + &Message{Message: &userUpdate, TypeName: "user", OpType: OpType_UPDATE_OP, FieldMaskPaths: []string{"PhoneNumber"}}, + ) + assert.NilError(t, err) + + types, err := NewTypeCatalog(Type{new(oplog_test.ReplayableTestUser), "user"}) + assert.NilError(t, err) + + var foundEntry Entry + err = tx.Where("id = ?", newLogEntry.Id).First(&foundEntry).Error + assert.NilError(t, err) + foundEntry.Cipherer = cipherer + err = foundEntry.DecryptData(context.Background()) + assert.NilError(t, err) + + err = foundEntry.Replay(context.Background(), &GormWriter{tx}, types, tableSuffix) + assert.NilError(t, err) + + var foundUser oplog_test.TestUser + err = tx.Where("id = ?", userCreate.Id).First(&foundUser).Error + assert.NilError(t, err) + + var foundReplayedUser oplog_test.TestUser + err = tx.Where("id = ?", userCreate.Id).First(&foundReplayedUser).Error + assert.NilError(t, err) + + assert.Assert(t, foundUser.Id == foundReplayedUser.Id) + assert.Assert(t, foundUser.Name == foundReplayedUser.Name && foundUser.Name == userName) + assert.Assert(t, foundUser.PhoneNumber == foundReplayedUser.PhoneNumber && foundReplayedUser.PhoneNumber == "867-5309") + assert.Assert(t, foundUser.Email == foundReplayedUser.Email && foundReplayedUser.Email == userName+"@hashicorp.com") + + tx.Commit() + }) + + t.Run("replay:delete", func(t *testing.T) { + + // we need to test delete replays now... + tx2 := db.Begin() + + ticket2, err := ticketer.GetTicket(ticketName) + assert.NilError(t, err) + + id4, err := uuid.GenerateUUID() + assert.NilError(t, err) + userName2 := "foo-" + id4 + // create a user that's replayable + userCreate2 := oplog_test.ReplayableTestUser{ + TestUser: oplog_test.TestUser{ + Name: userName2, + }, + } + err = tx2.Create(&userCreate2).Error + assert.NilError(t, err) + + deleteUser2 := oplog_test.ReplayableTestUser{ + TestUser: oplog_test.TestUser{ + Id: userCreate2.Id, + }, + } + err = tx2.Delete(&deleteUser2).Error + assert.NilError(t, err) + + newLogEntry2, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + err = newLogEntry2.WriteEntryWith(context.Background(), &GormWriter{tx2}, ticket2, + &Message{Message: &userCreate2, TypeName: "user", OpType: OpType_CREATE_OP}, + &Message{Message: &deleteUser2, TypeName: "user", OpType: OpType_DELETE_OP}, + ) + assert.NilError(t, err) + + var foundEntry2 Entry + err = tx2.Where("id = ?", newLogEntry2.Id).First(&foundEntry2).Error + assert.NilError(t, err) + foundEntry2.Cipherer = cipherer + err = foundEntry2.DecryptData(context.Background()) + assert.NilError(t, err) + + types, err := NewTypeCatalog(Type{new(oplog_test.ReplayableTestUser), "user"}) + assert.NilError(t, err) + + err = foundEntry2.Replay(context.Background(), &GormWriter{tx2}, types, tableSuffix) + assert.NilError(t, err) + + var foundUser2 oplog_test.TestUser + err = tx2.Where("id = ?", userCreate2.Id).First(&foundUser2).Error + assert.Assert(t, err == gorm.ErrRecordNotFound) + + var foundReplayedUser2 oplog_test.TestUser + err = tx2.Where("id = ?", userCreate2.Id).First(&foundReplayedUser2).Error + assert.Assert(t, err == gorm.ErrRecordNotFound) + + tx2.Commit() + }) +} + +// Test_WriteEntryWith provides unit tests for oplog.WriteEntryWith +func Test_WriteEntryWith(t *testing.T) { + t.Parallel() + startTest() + cleanup, url := setup(t) + defer cleanup() + defer completeTest() // must come after the "defer cleanup()" + db, err := test_dbconn(url) + assert.NilError(t, err) + defer db.Close() + + cipherer := initWrapper(t) + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + u := oplog_test.TestUser{ + Name: "foo-" + id, + } + t.Log(&u) + + id2, err := uuid.GenerateUUID() + assert.NilError(t, err) + u2 := oplog_test.TestUser{ + Name: "foo-" + id2, + } + t.Log(&u2) + + ticketName, err := uuid.GenerateUUID() + assert.NilError(t, err) + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + assert.NilError(t, err) + + err = ticketer.InitTicket(ticketName) + assert.NilError(t, err) + ticket, err := ticketer.GetTicket(ticketName) + assert.NilError(t, err) + + t.Run("successful", func(t *testing.T) { + newLogEntry, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + err = newLogEntry.WriteEntryWith(context.Background(), &GormWriter{db}, ticket, + &Message{Message: &u, TypeName: "user", OpType: OpType_CREATE_OP}, + &Message{Message: &u2, TypeName: "user", OpType: OpType_CREATE_OP}) + assert.NilError(t, err) + + var foundEntry Entry + err = db.Where("id = ?", newLogEntry.Id).First(&foundEntry).Error + assert.NilError(t, err) + foundEntry.Cipherer = cipherer + types, err := NewTypeCatalog(Type{new(oplog_test.TestUser), "user"}) + assert.NilError(t, err) + err = foundEntry.DecryptData(context.Background()) + assert.NilError(t, err) + foundUsers, err := foundEntry.UnmarshalData(types) + assert.NilError(t, err) + assert.Assert(t, foundUsers[0].Message.(*oplog_test.TestUser).Name == u.Name) + }) + t.Run("nil writer", func(t *testing.T) { + newLogEntry, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + err = newLogEntry.WriteEntryWith(context.Background(), nil, ticket, + &Message{Message: &u, TypeName: "user", OpType: OpType_CREATE_OP}, + &Message{Message: &u2, TypeName: "user", OpType: OpType_CREATE_OP}) + assert.Equal(t, err.Error(), "bad writer") + }) + t.Run("nil ticket", func(t *testing.T) { + newLogEntry, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + err = newLogEntry.WriteEntryWith(context.Background(), &GormWriter{db}, nil, + &Message{Message: &u, TypeName: "user", OpType: OpType_CREATE_OP}, + &Message{Message: &u2, TypeName: "user", OpType: OpType_CREATE_OP}) + assert.Equal(t, err.Error(), "bad ticket") + }) + t.Run("nil ticket", func(t *testing.T) { + newLogEntry, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + err = newLogEntry.WriteEntryWith(context.Background(), &GormWriter{db}, ticket, nil) + assert.Equal(t, err.Error(), "bad message") + }) +} + +// Test_TicketSerialization provides unit tests for making sure oplog.Tickets properly serialize writes to oplog entries +func Test_TicketSerialization(t *testing.T) { + startTest() + t.Parallel() + cleanup, url := setup(t) + defer cleanup() + defer completeTest() // must come after the "defer cleanup()" + db, err := test_dbconn(url) + assert.NilError(t, err) + defer db.Close() + + ticketName, err := uuid.GenerateUUID() + assert.NilError(t, err) + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + assert.NilError(t, err) + + // in it's own transaction, init the ticket + _ = ticketer.InitTicket(ticketName) + + cipherer := initWrapper(t) + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + firstTx := db.Begin() + firstUser := oplog_test.TestUser{ + Name: "foo-" + id, + } + err = firstTx.Create(&firstUser).Error + assert.NilError(t, err) + firstTicket, err := ticketer.GetTicket(ticketName) + assert.NilError(t, err) + + firstQueue := Queue{} + err = firstQueue.Add(&firstUser, "user", OpType_CREATE_OP) + assert.NilError(t, err) + + firstLogEntry, err := NewEntry( + "test-users", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + + firstLogEntry.Data = firstQueue.Bytes() + id2, err := uuid.GenerateUUID() + assert.NilError(t, err) + secondTx := db.Begin() + secondUser := oplog_test.TestUser{ + Name: "foo-" + id2, + } + err = secondTx.Create(&secondUser).Error + assert.NilError(t, err) + secondTicket, err := ticketer.GetTicket(ticketName) + assert.NilError(t, err) + + secondQueue := Queue{} + err = secondQueue.Add(&secondUser, "user", OpType_CREATE_OP) + assert.NilError(t, err) + + secondLogEntry, err := NewEntry( + "foobar", + Metadata{ + "key-only": nil, + "deployment": []string{"amex"}, + "project": []string{"central-info-systems", "local-info-systems"}, + }, + cipherer, + ticketer, + ) + assert.NilError(t, err) + secondLogEntry.Data = secondQueue.Bytes() + + err = secondLogEntry.Write(context.Background(), &GormWriter{secondTx}, secondTicket) + assert.NilError(t, err) + assert.Assert(t, secondLogEntry.Id != 0) + err = secondTx.Commit().Error + assert.NilError(t, err) + assert.Assert(t, secondLogEntry.Id != 0) + + err = firstLogEntry.Write(context.Background(), &GormWriter{firstTx}, firstTicket) + if err != nil { + firstTx.Rollback() + } else { + firstTx.Commit() + t.Error("should have failed to write firstLogEntry") + } +} + +// initDbInDocker initializes postgres within dockertest for the unit tests +func initDbInDocker(t *testing.T) (cleanup func(), retURL string, err error) { + if os.Getenv("PG_URL") != "" { + initTestStore(t, func() {}, os.Getenv("PG_URL")) + return func() {}, os.Getenv("PG_URL"), nil + } + pool, err := dockertest.NewPool("") + if err != nil { + return func() {}, "", fmt.Errorf("could not connect to docker: %w", err) + } + + resource, err := pool.Run("postgres", "latest", []string{"POSTGRES_PASSWORD=secret", "POSTGRES_DB=watchtower"}) + if err != nil { + return func() {}, "", fmt.Errorf("could not start resource: %w", err) + } + + c := func() { + cleanupResource(t, pool, resource) + } + + url := fmt.Sprintf("postgres://postgres:secret@localhost:%s?sslmode=disable", resource.GetPort("5432/tcp")) + + if err := pool.Retry(func() error { + db, err := sql.Open("postgres", url) + if err != nil { + return fmt.Errorf("error opening postgres dev container: %w", err) + } + + if err := db.Ping(); err != nil { + return err + } + defer db.Close() + return nil + }); err != nil { + return func() {}, "", fmt.Errorf("could not connect to docker: %w", err) + } + initTestStore(t, c, url) + return c, url, nil +} + +// initWrapper initializes an AEAD wrapping.Wrapper for testing the oplog +func initWrapper(t *testing.T) wrapping.Wrapper { + rootKey := make([]byte, 32) + n, err := rand.Read(rootKey) + if err != nil { + t.Fatal(err) + } + if n != 32 { + t.Fatal(n) + } + root := aead.NewWrapper(nil) + if err := root.SetAESGCMKeyBytes(rootKey); err != nil { + t.Fatal(err) + } + return root +} + +// initTestStore will execute the migrations needed to initialize the store for tests +func initTestStore(t *testing.T, cleanup func(), url string) { + // run migrations + m, err := migrate.New("file://migrations/postgres", url) + if err != nil { + cleanup() + t.Fatalf("Error creating migrations: %s", err) + } + if err := m.Up(); err != nil && err != migrate.ErrNoChange { + cleanup() + t.Fatalf("Error running migrations: %s", err) + } +} + +// cleanupResource will clean up the dockertest resources (postgres) +func cleanupResource(t *testing.T, pool *dockertest.Pool, resource *dockertest.Resource) { + waitForTests() + var err error + for i := 0; i < 10; i++ { + err = pool.Purge(resource) + if err == nil { + return + } + } + + if strings.Contains(err.Error(), "No such container") { + return + } + t.Fatalf("Failed to cleanup local container: %s", err) +} + +func test_dbconn(url string) (*gorm.DB, error) { + return gorm.Open("postgres", url) +} diff --git a/internal/oplog/oplog_test/db.go b/internal/oplog/oplog_test/db.go new file mode 100644 index 000000000..db5145226 --- /dev/null +++ b/internal/oplog/oplog_test/db.go @@ -0,0 +1,48 @@ +// Package oplog_test provides some gorm helper funcs for testing oplog database integrations +package oplog_test + +import "github.com/jinzhu/gorm" + +// Init will use gorm migrations to init tables for test models +func Init(db *gorm.DB) { + db.AutoMigrate(&TestUser{}) + db.AutoMigrate(&TestCar{}) + db.AutoMigrate(&TestRental{}) +} + +// Reinit will use gorm to drop then init tables for test models +func Reinit(db *gorm.DB) { + db.DropTableIfExists(&TestUser{}) + db.DropTableIfExists(&TestCar{}) + db.DropTableIfExists(&TestRental{}) + Init(db) +} + +// ReplayableTestUser is simply that: a user we can replay for tests +// the big diff is that it supports overriding the table name +type ReplayableTestUser struct { + TestUser + Table string `gorm:"-"` +} + +// TableName overrides the table name for the test user model +func (u *ReplayableTestUser) TableName() string { + if u.Table != "" { + return u.Table + } + return "oplog_test_user" +} + +func (u *ReplayableTestUser) SetTableName(name string) { + if name != "" { + u.Table = name + } +} + +func (*TestUser) TableName() string { return "oplog_test_user" } + +// TableName overrides the table name for the test car model +func (*TestCar) TableName() string { return "oplog_test_car" } + +// TableName overrids the table name for the test rental model +func (*TestRental) TableName() string { return "oplog_test_rental" } diff --git a/internal/oplog/oplog_test/oplog_test.pb.go b/internal/oplog/oplog_test/oplog_test.pb.go new file mode 100644 index 000000000..6086ac931 --- /dev/null +++ b/internal/oplog/oplog_test/oplog_test.pb.go @@ -0,0 +1,342 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.20.1 +// protoc v3.11.4 +// source: internal/oplog/oplog_test/oplog_test.proto + +// define a test proto package + +package oplog_test + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +// TestUser for gorm test user model +type TestUser struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // @inject_tag: gorm:"primary_key" + Id uint32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty" gorm:"primary_key"` + Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"` + PhoneNumber string `protobuf:"bytes,5,opt,name=phone_number,json=phoneNumber,proto3" json:"phone_number,omitempty"` + Email string `protobuf:"bytes,6,opt,name=email,proto3" json:"email,omitempty"` +} + +func (x *TestUser) Reset() { + *x = TestUser{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_oplog_oplog_test_oplog_test_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TestUser) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TestUser) ProtoMessage() {} + +func (x *TestUser) ProtoReflect() protoreflect.Message { + mi := &file_internal_oplog_oplog_test_oplog_test_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TestUser.ProtoReflect.Descriptor instead. +func (*TestUser) Descriptor() ([]byte, []int) { + return file_internal_oplog_oplog_test_oplog_test_proto_rawDescGZIP(), []int{0} +} + +func (x *TestUser) GetId() uint32 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *TestUser) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *TestUser) GetPhoneNumber() string { + if x != nil { + return x.PhoneNumber + } + return "" +} + +func (x *TestUser) GetEmail() string { + if x != nil { + return x.Email + } + return "" +} + +// TestCar for gorm test car model +type TestCar struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // @inject_tag: gorm:"primary_key" + Id uint32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty" gorm:"primary_key"` + Model string `protobuf:"bytes,4,opt,name=model,proto3" json:"model,omitempty"` + Mpg int32 `protobuf:"varint,5,opt,name=mpg,proto3" json:"mpg,omitempty"` +} + +func (x *TestCar) Reset() { + *x = TestCar{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_oplog_oplog_test_oplog_test_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TestCar) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TestCar) ProtoMessage() {} + +func (x *TestCar) ProtoReflect() protoreflect.Message { + mi := &file_internal_oplog_oplog_test_oplog_test_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TestCar.ProtoReflect.Descriptor instead. +func (*TestCar) Descriptor() ([]byte, []int) { + return file_internal_oplog_oplog_test_oplog_test_proto_rawDescGZIP(), []int{1} +} + +func (x *TestCar) GetId() uint32 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *TestCar) GetModel() string { + if x != nil { + return x.Model + } + return "" +} + +func (x *TestCar) GetMpg() int32 { + if x != nil { + return x.Mpg + } + return 0 +} + +// TestRental for gorm test rental model +type TestRental struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + UserId uint32 `protobuf:"varint,1,opt,name=user_id,json=userId,proto3" json:"user_id,omitempty"` + CarId uint32 `protobuf:"varint,2,opt,name=car_id,json=carId,proto3" json:"car_id,omitempty"` +} + +func (x *TestRental) Reset() { + *x = TestRental{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_oplog_oplog_test_oplog_test_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TestRental) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TestRental) ProtoMessage() {} + +func (x *TestRental) ProtoReflect() protoreflect.Message { + mi := &file_internal_oplog_oplog_test_oplog_test_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TestRental.ProtoReflect.Descriptor instead. +func (*TestRental) Descriptor() ([]byte, []int) { + return file_internal_oplog_oplog_test_oplog_test_proto_rawDescGZIP(), []int{2} +} + +func (x *TestRental) GetUserId() uint32 { + if x != nil { + return x.UserId + } + return 0 +} + +func (x *TestRental) GetCarId() uint32 { + if x != nil { + return x.CarId + } + return 0 +} + +var File_internal_oplog_oplog_test_oplog_test_proto protoreflect.FileDescriptor + +var file_internal_oplog_oplog_test_oplog_test_proto_rawDesc = []byte{ + 0x0a, 0x2a, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6f, 0x70, 0x6c, 0x6f, 0x67, + 0x2f, 0x6f, 0x70, 0x6c, 0x6f, 0x67, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x2f, 0x6f, 0x70, 0x6c, 0x6f, + 0x67, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x2d, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x77, 0x61, 0x74, 0x63, 0x68, 0x74, 0x6f, 0x77, + 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x6f, 0x70, + 0x6c, 0x6f, 0x67, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x76, 0x31, 0x22, 0x67, 0x0a, 0x08, 0x54, + 0x65, 0x73, 0x74, 0x55, 0x73, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x70, + 0x68, 0x6f, 0x6e, 0x65, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x70, 0x68, 0x6f, 0x6e, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x14, + 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, + 0x6d, 0x61, 0x69, 0x6c, 0x22, 0x41, 0x0a, 0x07, 0x54, 0x65, 0x73, 0x74, 0x43, 0x61, 0x72, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x14, 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x70, 0x67, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x03, 0x6d, 0x70, 0x67, 0x22, 0x3c, 0x0a, 0x0a, 0x54, 0x65, 0x73, 0x74, 0x52, + 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x15, + 0x0a, 0x06, 0x63, 0x61, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, + 0x63, 0x61, 0x72, 0x49, 0x64, 0x42, 0x46, 0x5a, 0x44, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x77, 0x61, + 0x74, 0x63, 0x68, 0x74, 0x6f, 0x77, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2f, 0x6f, 0x70, 0x6c, 0x6f, 0x67, 0x2f, 0x6f, 0x70, 0x6c, 0x6f, 0x67, 0x5f, 0x74, 0x65, + 0x73, 0x74, 0x3b, 0x6f, 0x70, 0x6c, 0x6f, 0x67, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_internal_oplog_oplog_test_oplog_test_proto_rawDescOnce sync.Once + file_internal_oplog_oplog_test_oplog_test_proto_rawDescData = file_internal_oplog_oplog_test_oplog_test_proto_rawDesc +) + +func file_internal_oplog_oplog_test_oplog_test_proto_rawDescGZIP() []byte { + file_internal_oplog_oplog_test_oplog_test_proto_rawDescOnce.Do(func() { + file_internal_oplog_oplog_test_oplog_test_proto_rawDescData = protoimpl.X.CompressGZIP(file_internal_oplog_oplog_test_oplog_test_proto_rawDescData) + }) + return file_internal_oplog_oplog_test_oplog_test_proto_rawDescData +} + +var file_internal_oplog_oplog_test_oplog_test_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_internal_oplog_oplog_test_oplog_test_proto_goTypes = []interface{}{ + (*TestUser)(nil), // 0: hashicorp.watchtower.controller.oplog.test.v1.TestUser + (*TestCar)(nil), // 1: hashicorp.watchtower.controller.oplog.test.v1.TestCar + (*TestRental)(nil), // 2: hashicorp.watchtower.controller.oplog.test.v1.TestRental +} +var file_internal_oplog_oplog_test_oplog_test_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_internal_oplog_oplog_test_oplog_test_proto_init() } +func file_internal_oplog_oplog_test_oplog_test_proto_init() { + if File_internal_oplog_oplog_test_oplog_test_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_internal_oplog_oplog_test_oplog_test_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TestUser); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_oplog_oplog_test_oplog_test_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TestCar); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_oplog_oplog_test_oplog_test_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TestRental); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_internal_oplog_oplog_test_oplog_test_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_internal_oplog_oplog_test_oplog_test_proto_goTypes, + DependencyIndexes: file_internal_oplog_oplog_test_oplog_test_proto_depIdxs, + MessageInfos: file_internal_oplog_oplog_test_oplog_test_proto_msgTypes, + }.Build() + File_internal_oplog_oplog_test_oplog_test_proto = out.File + file_internal_oplog_oplog_test_oplog_test_proto_rawDesc = nil + file_internal_oplog_oplog_test_oplog_test_proto_goTypes = nil + file_internal_oplog_oplog_test_oplog_test_proto_depIdxs = nil +} diff --git a/internal/oplog/oplog_test/oplog_test.proto b/internal/oplog/oplog_test/oplog_test.proto new file mode 100644 index 000000000..a728228ce --- /dev/null +++ b/internal/oplog/oplog_test/oplog_test.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +// define a test proto package +package hashicorp.watchtower.controller.oplog.test.v1; +option go_package = "github.com/hashicorp/watchtower/internal/oplog/oplog_test;oplog_test"; + +// TestUser for gorm test user model +message TestUser { + // @inject_tag: gorm:"primary_key" + uint32 id = 1; + string name = 4; + string phone_number = 5; + string email = 6; +} +// TestCar for gorm test car model +message TestCar { + // @inject_tag: gorm:"primary_key" + uint32 id = 1; + string model = 4; + int32 mpg = 5; +} +// TestRental for gorm test rental model +message TestRental { + uint32 user_id = 1; + uint32 car_id = 2; +} \ No newline at end of file diff --git a/internal/oplog/options.go b/internal/oplog/options.go new file mode 100644 index 000000000..7c0ceb3e0 --- /dev/null +++ b/internal/oplog/options.go @@ -0,0 +1,42 @@ +package oplog + +// GetOpts - iterate the inbound Options and return a struct +func GetOpts(opt ...Option) Options { + opts := getDefaultOptions() + for _, o := range opt { + o(opts) + } + return opts +} + +// Option - how Options are passed as arguments +type Option func(Options) + +// Options = how options are represented +type Options map[string]interface{} + +func getDefaultOptions() Options { + return Options{ + optionWithFieldMaskPaths: []string{}, + optionWithAggregateNames: false, + } +} + +const optionWithFieldMaskPaths = "optionWithFieldMaskPaths" + +// WithFieldMaskPaths represents an optional set of symbolic field paths (for example: "f.a", "f.b.d") used +// to specify a subset of fields that should be updated. (see google.golang.org/genproto/protobuf/field_mask) +func WithFieldMaskPaths(fieldMaskPaths []string) Option { + return func(o Options) { + o[optionWithFieldMaskPaths] = fieldMaskPaths + } +} + +const optionWithAggregateNames = "optionWithAggregateNames" + +// WithAggregateNames enables/disables the use of multiple aggregate names for Ticketers +func WithAggregateNames(enabled bool) Option { + return func(o Options) { + o[optionWithAggregateNames] = enabled + } +} diff --git a/internal/oplog/queue.go b/internal/oplog/queue.go new file mode 100644 index 000000000..36b62d4a7 --- /dev/null +++ b/internal/oplog/queue.go @@ -0,0 +1,91 @@ +package oplog + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "sync" + + "google.golang.org/genproto/protobuf/field_mask" + "google.golang.org/protobuf/proto" +) + +// Queue provides a FIFO queue +type Queue struct { + // Buffer for the queue + bytes.Buffer + // Catalog provides a TypeCatalog for the types added to the queue + Catalog *TypeCatalog + + mx sync.Mutex +} + +// Add pb message to queue +func (r *Queue) Add(m proto.Message, typeName string, t OpType, opt ...Option) error { + // we're not checking the Catalog for nil, since it's not used + // when Adding messages to the queue + opts := GetOpts(opt...) + withPaths := opts[optionWithFieldMaskPaths].([]string) + + value, err := proto.Marshal(m) + if err != nil { + return fmt.Errorf("error marshaling add parameter: %w", err) + } + msg := &AnyOperation{ + TypeName: typeName, + Value: value, + OperationType: t, + FieldMask: &field_mask.FieldMask{Paths: withPaths}, + } + data, err := proto.Marshal(msg) + if err != nil { + return fmt.Errorf("error marhaling the msg for Add: %w", err) + } + r.mx.Lock() + defer r.mx.Unlock() + err = binary.Write(r, binary.LittleEndian, uint32(len(data))) + if err != nil { + return err + } + n, err := r.Write(data) + if err != nil { + return fmt.Errorf("error writing to queue buffer: %w", err) + } + if n != len(data) { + return fmt.Errorf("error writing to queue buffer (incorrect number of bytes %d of %d)", n, len(data)) + } + return nil +} + +// Remove pb message from the queue and EOF if empty +func (r *Queue) Remove() (proto.Message, OpType, []string, error) { + if r.Catalog == nil { + return nil, OpType_UNKNOWN_OP, nil, errors.New("remove Catalog is nil") + } + r.mx.Lock() + defer r.mx.Unlock() + var n uint32 + err := binary.Read(r, binary.LittleEndian, &n) + if err != nil { + return nil, 0, nil, err // intentionally not wrapping error so client can test for sentinel EOF error + } + data := r.Next(int(n)) + msg := new(AnyOperation) + err = proto.Unmarshal(data, msg) + if err != nil { + return nil, 0, nil, fmt.Errorf("error marshaling the msg for Remove: %w", err) + } + if msg.Value == nil { + return nil, 0, nil, nil + } + any, err := r.Catalog.Get(msg.TypeName) + if err != nil { + return nil, 0, nil, fmt.Errorf("error getting the TypeName for Remove: %w", err) + } + pm := any.(proto.Message) + if err = proto.Unmarshal(msg.Value, pm); err != nil { + return nil, 0, nil, fmt.Errorf("error unmarshaling the value for Remove: %w", err) + } + return pm, msg.OperationType, msg.FieldMask.GetPaths(), nil +} diff --git a/internal/oplog/queue_test.go b/internal/oplog/queue_test.go new file mode 100644 index 000000000..87629c413 --- /dev/null +++ b/internal/oplog/queue_test.go @@ -0,0 +1,89 @@ +package oplog + +import ( + reflect "reflect" + "testing" + + "github.com/hashicorp/watchtower/internal/oplog/oplog_test" + "google.golang.org/protobuf/proto" + "gotest.tools/assert" +) + +// Test_Queue provides basic tests for the Queue type +func Test_Queue(t *testing.T) { + t.Parallel() + types, err := NewTypeCatalog( + Type{new(oplog_test.TestUser), "user"}, + Type{new(oplog_test.TestCar), "car"}, + Type{new(oplog_test.TestRental), "rental"}, + ) + assert.NilError(t, err) + queue := Queue{Catalog: types} + + user := &oplog_test.TestUser{ + Id: 1, + Name: "Alice", + PhoneNumber: "867-5309", + Email: "alice@bob.com", + } + car := &oplog_test.TestCar{ + Id: 2, + Model: "Jeep", + Mpg: 25, + } + rental := &oplog_test.TestRental{ + UserId: 1, + CarId: 2, + } + + userUpdate := &oplog_test.TestUser{ + Id: 1, + Name: "bob", + Email: "bob@alice.com", + } + t.Run("add/remove", func(t *testing.T) { + err = queue.Add(user, "user", OpType_CREATE_OP) + assert.NilError(t, err) + err = queue.Add(car, "car", OpType_CREATE_OP) + assert.NilError(t, err) + err = queue.Add(rental, "rental", OpType_CREATE_OP) + assert.NilError(t, err) + err = queue.Add(userUpdate, "user", OpType_UPDATE_OP, WithFieldMaskPaths([]string{"Name", "Email"})) + assert.NilError(t, err) + + queuedUser, ty, fm, err := queue.Remove() + assert.NilError(t, err) + assert.Assert(t, proto.Equal(user, queuedUser)) + assert.Assert(t, ty == OpType_CREATE_OP) + assert.Assert(t, fm == nil) + + queuedCar, ty, fm, err := queue.Remove() + assert.NilError(t, err) + assert.Assert(t, proto.Equal(car, queuedCar)) + assert.Assert(t, ty == OpType_CREATE_OP) + assert.Assert(t, fm == nil) + + queuedRental, ty, fm, err := queue.Remove() + assert.NilError(t, err) + assert.Assert(t, proto.Equal(rental, queuedRental)) + assert.Assert(t, ty == OpType_CREATE_OP) + assert.Assert(t, fm == nil) + + queuedUserUpdate, ty, fm, err := queue.Remove() + assert.NilError(t, err) + assert.Assert(t, proto.Equal(userUpdate, queuedUserUpdate)) + assert.Assert(t, ty == OpType_UPDATE_OP) + assert.Assert(t, reflect.DeepEqual(fm, []string{"Name", "Email"})) + }) + t.Run("valid with nil type catalog", func(t *testing.T) { + queue := Queue{} + assert.NilError(t, queue.Add(user, "user", OpType_CREATE_OP)) + }) + t.Run("error with nil type catalog", func(t *testing.T) { + queue := Queue{} + _, _, _, err := queue.Remove() + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "remove Catalog is nil") + }) + +} diff --git a/internal/oplog/replayable.go b/internal/oplog/replayable.go new file mode 100644 index 000000000..b403167c4 --- /dev/null +++ b/internal/oplog/replayable.go @@ -0,0 +1,8 @@ +package oplog + +// Replayable defines an interface for messages that can be replayed from the oplog entries. we need to be +// able to replay into different table names. +type ReplayableMessage interface { + TableName() string + SetTableName(name string) +} diff --git a/internal/oplog/store/doc.go b/internal/oplog/store/doc.go new file mode 100644 index 000000000..a9e647f44 --- /dev/null +++ b/internal/oplog/store/doc.go @@ -0,0 +1,2 @@ +// Package store provides storage types/behavior for the oplog +package store diff --git a/internal/oplog/store/oplog.pb.go b/internal/oplog/store/oplog.pb.go new file mode 100644 index 000000000..5d1ac538a --- /dev/null +++ b/internal/oplog/store/oplog.pb.go @@ -0,0 +1,552 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.20.1 +// protoc v3.11.4 +// source: internal/oplog/store/oplog.proto + +// define an oplog package within the watchtower controller + +package store + +import ( + proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +// Timestamp for oplog messages. We've defined a new local type wrapper +// of google.protobuf.Timestamp so we can implement sql.Scanner and sql.Valuer +// interfaces. See: +// https://golang.org/pkg/database/sql/#Scanner +// https://golang.org/pkg/database/sql/driver/#Valuer +type Timestamp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Timestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` +} + +func (x *Timestamp) Reset() { + *x = Timestamp{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_oplog_store_oplog_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Timestamp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Timestamp) ProtoMessage() {} + +func (x *Timestamp) ProtoReflect() protoreflect.Message { + mi := &file_internal_oplog_store_oplog_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Timestamp.ProtoReflect.Descriptor instead. +func (*Timestamp) Descriptor() ([]byte, []int) { + return file_internal_oplog_store_oplog_proto_rawDescGZIP(), []int{0} +} + +func (x *Timestamp) GetTimestamp() *timestamp.Timestamp { + if x != nil { + return x.Timestamp + } + return nil +} + +// Entry provides a message for oplog entries that's compatible with gorm +type Entry struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // @inject_tag: gorm:"primary_key" + Id uint32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty" gorm:"primary_key"` + // @inject_tag: gorm:"not_null;type:TIMESTAMP" + CreateTime *Timestamp `protobuf:"bytes,2,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty" gorm:"not_null;type:TIMESTAMP"` + // @inject_tag: gorm:"not_null;type:TIMESTAMP" + UpdateTime *Timestamp `protobuf:"bytes,3,opt,name=update_time,json=updateTime,proto3" json:"update_time,omitempty" gorm:"not_null;type:TIMESTAMP"` + // @inject_tag: gorm:"not_null" + Version string `protobuf:"bytes,4,opt,name=version,proto3" json:"version,omitempty" gorm:"not_null"` + // @inject_tat: gorm:"not_null" + AggregateName string `protobuf:"bytes,5,opt,name=aggregate_name,json=aggregateName,proto3" json:"aggregate_name,omitempty"` + // one to many relationship + // @inject_tag: gorm:"foreignkey:entry_id" + Metadata []*Metadata `protobuf:"bytes,6,rep,name=metadata,proto3" json:"metadata,omitempty" gorm:"foreignkey:entry_id"` + // @inject_tat: gorm:"not_null" + Data []byte `protobuf:"bytes,7,opt,name=data,proto3" json:"data,omitempty"` +} + +func (x *Entry) Reset() { + *x = Entry{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_oplog_store_oplog_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Entry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Entry) ProtoMessage() {} + +func (x *Entry) ProtoReflect() protoreflect.Message { + mi := &file_internal_oplog_store_oplog_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Entry.ProtoReflect.Descriptor instead. +func (*Entry) Descriptor() ([]byte, []int) { + return file_internal_oplog_store_oplog_proto_rawDescGZIP(), []int{1} +} + +func (x *Entry) GetId() uint32 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *Entry) GetCreateTime() *Timestamp { + if x != nil { + return x.CreateTime + } + return nil +} + +func (x *Entry) GetUpdateTime() *Timestamp { + if x != nil { + return x.UpdateTime + } + return nil +} + +func (x *Entry) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *Entry) GetAggregateName() string { + if x != nil { + return x.AggregateName + } + return "" +} + +func (x *Entry) GetMetadata() []*Metadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *Entry) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +// Metadata provides a message for oplog metadata that's compatible with gorm +type Metadata struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // @inject_tag: gorm:"primary_key" + Id uint32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty" gorm:"primary_key"` + // @inject_tag: gorm:"not_null;type:TIMESTAMP" + CreateTime *Timestamp `protobuf:"bytes,2,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty" gorm:"not_null;type:TIMESTAMP"` // @inject_tag: gorm:"not_nul + EntryId uint32 `protobuf:"varint,3,opt,name=entry_id,json=entryId,proto3" json:"entry_id,omitempty"` + // @inject_tag: gorm:"foreignkey:EntryId" + Entry *Entry `protobuf:"bytes,4,opt,name=entry,proto3" json:"entry,omitempty" gorm:"foreignkey:EntryId"` + // @inject_tag: gorm:"not_null" + Key string `protobuf:"bytes,5,opt,name=key,proto3" json:"key,omitempty" gorm:"not_null"` + // @inject_tag: gorm:"not_null" + Value string `protobuf:"bytes,6,opt,name=value,proto3" json:"value,omitempty" gorm:"not_null"` +} + +func (x *Metadata) Reset() { + *x = Metadata{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_oplog_store_oplog_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Metadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Metadata) ProtoMessage() {} + +func (x *Metadata) ProtoReflect() protoreflect.Message { + mi := &file_internal_oplog_store_oplog_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Metadata.ProtoReflect.Descriptor instead. +func (*Metadata) Descriptor() ([]byte, []int) { + return file_internal_oplog_store_oplog_proto_rawDescGZIP(), []int{2} +} + +func (x *Metadata) GetId() uint32 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *Metadata) GetCreateTime() *Timestamp { + if x != nil { + return x.CreateTime + } + return nil +} + +func (x *Metadata) GetEntryId() uint32 { + if x != nil { + return x.EntryId + } + return 0 +} + +func (x *Metadata) GetEntry() *Entry { + if x != nil { + return x.Entry + } + return nil +} + +func (x *Metadata) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *Metadata) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +// Ticket provides a message for oplog tickets that's compatible with gorm +type Ticket struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // @inject_tag: gorm:"primary_key" + Id uint32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty" gorm:"primary_key"` + // @inject_tag: gorm:"not_null;type:TIMESTAMP" + CreateTime *Timestamp `protobuf:"bytes,2,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty" gorm:"not_null;type:TIMESTAMP"` + // @inject_tag: gorm:"not_null;type:TIMESTAMP" + UpdateTime *Timestamp `protobuf:"bytes,3,opt,name=update_time,json=updateTime,proto3" json:"update_time,omitempty" gorm:"not_null;type:TIMESTAMP"` + // @inject_tat: gorm:"not_null" + Name string `protobuf:"bytes,6,opt,name=name,proto3" json:"name,omitempty"` + // @inject_tat: gorm:"not_null;default:'1'" + Version uint32 `protobuf:"varint,7,opt,name=version,proto3" json:"version,omitempty"` +} + +func (x *Ticket) Reset() { + *x = Ticket{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_oplog_store_oplog_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Ticket) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Ticket) ProtoMessage() {} + +func (x *Ticket) ProtoReflect() protoreflect.Message { + mi := &file_internal_oplog_store_oplog_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Ticket.ProtoReflect.Descriptor instead. +func (*Ticket) Descriptor() ([]byte, []int) { + return file_internal_oplog_store_oplog_proto_rawDescGZIP(), []int{3} +} + +func (x *Ticket) GetId() uint32 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *Ticket) GetCreateTime() *Timestamp { + if x != nil { + return x.CreateTime + } + return nil +} + +func (x *Ticket) GetUpdateTime() *Timestamp { + if x != nil { + return x.UpdateTime + } + return nil +} + +func (x *Ticket) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Ticket) GetVersion() uint32 { + if x != nil { + return x.Version + } + return 0 +} + +var File_internal_oplog_store_oplog_proto protoreflect.FileDescriptor + +var file_internal_oplog_store_oplog_proto_rawDesc = []byte{ + 0x0a, 0x20, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6f, 0x70, 0x6c, 0x6f, 0x67, + 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x6f, 0x70, 0x6c, 0x6f, 0x67, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x12, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x77, 0x61, + 0x74, 0x63, 0x68, 0x74, 0x6f, 0x77, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x2e, 0x6f, 0x70, 0x6c, 0x6f, 0x67, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, + 0x76, 0x31, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x22, 0x45, 0x0a, 0x09, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0xfa, 0x02, 0x0a, 0x05, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x5a, 0x0a, 0x0b, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x77, 0x61, 0x74, 0x63, 0x68, 0x74, 0x6f, 0x77, 0x65, 0x72, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x6f, 0x70, 0x6c, 0x6f, + 0x67, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, + 0x12, 0x5a, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x77, 0x61, 0x74, 0x63, 0x68, 0x74, 0x6f, 0x77, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x6f, 0x70, 0x6c, 0x6f, 0x67, 0x2e, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, + 0x61, 0x74, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, + 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x54, 0x0a, + 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x77, 0x61, 0x74, 0x63, + 0x68, 0x74, 0x6f, 0x77, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, + 0x72, 0x2e, 0x6f, 0x70, 0x6c, 0x6f, 0x67, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x86, 0x02, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x5a, 0x0a, 0x0b, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x77, 0x61, 0x74, 0x63, 0x68, 0x74, 0x6f, 0x77, 0x65, 0x72, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x6f, 0x70, 0x6c, 0x6f, + 0x67, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, + 0x12, 0x19, 0x0a, 0x08, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x49, 0x64, 0x12, 0x4b, 0x0a, 0x05, 0x65, + 0x6e, 0x74, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x77, 0x61, 0x74, 0x63, 0x68, 0x74, 0x6f, 0x77, 0x65, + 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x6f, 0x70, 0x6c, + 0x6f, 0x67, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x22, 0xfe, 0x01, 0x0a, 0x06, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x5a, 0x0a, 0x0b, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x77, 0x61, 0x74, + 0x63, 0x68, 0x74, 0x6f, 0x77, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, + 0x65, 0x72, 0x2e, 0x6f, 0x70, 0x6c, 0x6f, 0x67, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x63, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x5a, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x77, 0x61, 0x74, 0x63, 0x68, 0x74, 0x6f, + 0x77, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x6f, + 0x70, 0x6c, 0x6f, 0x67, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, + 0x69, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x77, 0x61, 0x74, 0x63, 0x68, 0x74, + 0x6f, 0x77, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6f, 0x70, + 0x6c, 0x6f, 0x67, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_internal_oplog_store_oplog_proto_rawDescOnce sync.Once + file_internal_oplog_store_oplog_proto_rawDescData = file_internal_oplog_store_oplog_proto_rawDesc +) + +func file_internal_oplog_store_oplog_proto_rawDescGZIP() []byte { + file_internal_oplog_store_oplog_proto_rawDescOnce.Do(func() { + file_internal_oplog_store_oplog_proto_rawDescData = protoimpl.X.CompressGZIP(file_internal_oplog_store_oplog_proto_rawDescData) + }) + return file_internal_oplog_store_oplog_proto_rawDescData +} + +var file_internal_oplog_store_oplog_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_internal_oplog_store_oplog_proto_goTypes = []interface{}{ + (*Timestamp)(nil), // 0: hashicorp.watchtower.controller.oplog.store.v1.Timestamp + (*Entry)(nil), // 1: hashicorp.watchtower.controller.oplog.store.v1.Entry + (*Metadata)(nil), // 2: hashicorp.watchtower.controller.oplog.store.v1.Metadata + (*Ticket)(nil), // 3: hashicorp.watchtower.controller.oplog.store.v1.Ticket + (*timestamp.Timestamp)(nil), // 4: google.protobuf.Timestamp +} +var file_internal_oplog_store_oplog_proto_depIdxs = []int32{ + 4, // 0: hashicorp.watchtower.controller.oplog.store.v1.Timestamp.timestamp:type_name -> google.protobuf.Timestamp + 0, // 1: hashicorp.watchtower.controller.oplog.store.v1.Entry.create_time:type_name -> hashicorp.watchtower.controller.oplog.store.v1.Timestamp + 0, // 2: hashicorp.watchtower.controller.oplog.store.v1.Entry.update_time:type_name -> hashicorp.watchtower.controller.oplog.store.v1.Timestamp + 2, // 3: hashicorp.watchtower.controller.oplog.store.v1.Entry.metadata:type_name -> hashicorp.watchtower.controller.oplog.store.v1.Metadata + 0, // 4: hashicorp.watchtower.controller.oplog.store.v1.Metadata.create_time:type_name -> hashicorp.watchtower.controller.oplog.store.v1.Timestamp + 1, // 5: hashicorp.watchtower.controller.oplog.store.v1.Metadata.entry:type_name -> hashicorp.watchtower.controller.oplog.store.v1.Entry + 0, // 6: hashicorp.watchtower.controller.oplog.store.v1.Ticket.create_time:type_name -> hashicorp.watchtower.controller.oplog.store.v1.Timestamp + 0, // 7: hashicorp.watchtower.controller.oplog.store.v1.Ticket.update_time:type_name -> hashicorp.watchtower.controller.oplog.store.v1.Timestamp + 8, // [8:8] is the sub-list for method output_type + 8, // [8:8] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name +} + +func init() { file_internal_oplog_store_oplog_proto_init() } +func file_internal_oplog_store_oplog_proto_init() { + if File_internal_oplog_store_oplog_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_internal_oplog_store_oplog_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Timestamp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_oplog_store_oplog_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Entry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_oplog_store_oplog_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Metadata); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_oplog_store_oplog_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Ticket); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_internal_oplog_store_oplog_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_internal_oplog_store_oplog_proto_goTypes, + DependencyIndexes: file_internal_oplog_store_oplog_proto_depIdxs, + MessageInfos: file_internal_oplog_store_oplog_proto_msgTypes, + }.Build() + File_internal_oplog_store_oplog_proto = out.File + file_internal_oplog_store_oplog_proto_rawDesc = nil + file_internal_oplog_store_oplog_proto_goTypes = nil + file_internal_oplog_store_oplog_proto_depIdxs = nil +} diff --git a/internal/oplog/store/oplog.pb.json.go b/internal/oplog/store/oplog.pb.json.go new file mode 100644 index 000000000..96992982e --- /dev/null +++ b/internal/oplog/store/oplog.pb.json.go @@ -0,0 +1,82 @@ +// Code generated by protoc-gen-go-json. DO NOT EDIT. +// source: oplog.proto + +package store + +import ( + "bytes" + + "github.com/golang/protobuf/jsonpb" +) + +// MarshalJSON implements json.Marshaler +func (msg *Timestamp) MarshalJSON() ([]byte, error) { + var buf bytes.Buffer + err := (&jsonpb.Marshaler{ + EnumsAsInts: false, + EmitDefaults: false, + OrigName: false, + }).Marshal(&buf, msg) + return buf.Bytes(), err +} + +// UnmarshalJSON implements json.Unmarshaler +func (msg *Timestamp) UnmarshalJSON(b []byte) error { + return (&jsonpb.Unmarshaler{ + AllowUnknownFields: false, + }).Unmarshal(bytes.NewReader(b), msg) +} + +// MarshalJSON implements json.Marshaler +func (msg *Entry) MarshalJSON() ([]byte, error) { + var buf bytes.Buffer + err := (&jsonpb.Marshaler{ + EnumsAsInts: false, + EmitDefaults: false, + OrigName: false, + }).Marshal(&buf, msg) + return buf.Bytes(), err +} + +// UnmarshalJSON implements json.Unmarshaler +func (msg *Entry) UnmarshalJSON(b []byte) error { + return (&jsonpb.Unmarshaler{ + AllowUnknownFields: false, + }).Unmarshal(bytes.NewReader(b), msg) +} + +// MarshalJSON implements json.Marshaler +func (msg *Metadata) MarshalJSON() ([]byte, error) { + var buf bytes.Buffer + err := (&jsonpb.Marshaler{ + EnumsAsInts: false, + EmitDefaults: false, + OrigName: false, + }).Marshal(&buf, msg) + return buf.Bytes(), err +} + +// UnmarshalJSON implements json.Unmarshaler +func (msg *Metadata) UnmarshalJSON(b []byte) error { + return (&jsonpb.Unmarshaler{ + AllowUnknownFields: false, + }).Unmarshal(bytes.NewReader(b), msg) +} + +// MarshalJSON implements json.Marshaler +func (msg *Ticket) MarshalJSON() ([]byte, error) { + var buf bytes.Buffer + err := (&jsonpb.Marshaler{ + EnumsAsInts: false, + EmitDefaults: false, + OrigName: false, + }).Marshal(&buf, msg) + return buf.Bytes(), err +} + +// UnmarshalJSON implements json.Unmarshaler +func (msg *Ticket) UnmarshalJSON(b []byte) error { + return (&jsonpb.Unmarshaler{ + AllowUnknownFields: false, + }).Unmarshal(bytes.NewReader(b), msg) +} diff --git a/internal/oplog/store/oplog.proto b/internal/oplog/store/oplog.proto new file mode 100644 index 000000000..9d600ee3f --- /dev/null +++ b/internal/oplog/store/oplog.proto @@ -0,0 +1,73 @@ +syntax = "proto3"; + +// define an oplog package within the watchtower controller +package hashicorp.watchtower.controller.oplog.store.v1; +option go_package = "github.com/hashicorp/watchtower/internal/oplog/store;store"; + +import "google/protobuf/timestamp.proto"; + +// Timestamp for oplog messages. We've defined a new local type wrapper +// of google.protobuf.Timestamp so we can implement sql.Scanner and sql.Valuer +// interfaces. See: +// https://golang.org/pkg/database/sql/#Scanner +// https://golang.org/pkg/database/sql/driver/#Valuer +message Timestamp { + google.protobuf.Timestamp timestamp = 1; +} + +// Entry provides a message for oplog entries that's compatible with gorm +message Entry { + // @inject_tag: gorm:"primary_key" + uint32 id = 1; + // @inject_tag: gorm:"not_null;type:TIMESTAMP" + Timestamp create_time = 2; + // @inject_tag: gorm:"not_null;type:TIMESTAMP" + Timestamp update_time = 3; + + // @inject_tag: gorm:"not_null" + string version = 4; + + // @inject_tat: gorm:"not_null" + string aggregate_name = 5; + + // one to many relationship + // @inject_tag: gorm:"foreignkey:entry_id" + repeated Metadata metadata = 6; + + // @inject_tat: gorm:"not_null" + bytes data = 7; +} + +// Metadata provides a message for oplog metadata that's compatible with gorm +message Metadata { + // @inject_tag: gorm:"primary_key" + uint32 id = 1; + // @inject_tag: gorm:"not_null;type:TIMESTAMP" + Timestamp create_time = 2; + // @inject_tag: gorm:"not_nul + + uint32 entry_id = 3; + // @inject_tag: gorm:"foreignkey:EntryId" + Entry entry = 4; + + // @inject_tag: gorm:"not_null" + string key = 5; + // @inject_tag: gorm:"not_null" + string value = 6; +} + +// Ticket provides a message for oplog tickets that's compatible with gorm +message Ticket { + // @inject_tag: gorm:"primary_key" + uint32 id = 1; + // @inject_tag: gorm:"not_null;type:TIMESTAMP" + Timestamp create_time = 2; + // @inject_tag: gorm:"not_null;type:TIMESTAMP" + Timestamp update_time = 3; + + // @inject_tat: gorm:"not_null" + string name = 6; + + // @inject_tat: gorm:"not_null;default:'1'" + uint32 version = 7; +} diff --git a/internal/oplog/store/oplog_entry.go b/internal/oplog/store/oplog_entry.go new file mode 100644 index 000000000..96efbc179 --- /dev/null +++ b/internal/oplog/store/oplog_entry.go @@ -0,0 +1,10 @@ +package store + +// TableName overrides the gorm database table for the store.Oplog messages +func (*Entry) TableName() string { return "oplog_entry" } + +// TableName overrides the gorm database table for the store.Ticket messages +func (*Ticket) TableName() string { return "oplog_ticket" } + +// TableName overrides the gorm database table for the store.Metadata messages +func (*Metadata) TableName() string { return "oplog_metadata" } diff --git a/internal/oplog/store/scanner_test.go b/internal/oplog/store/scanner_test.go new file mode 100644 index 000000000..022648123 --- /dev/null +++ b/internal/oplog/store/scanner_test.go @@ -0,0 +1,73 @@ +package store + +import ( + "reflect" + "testing" + "time" + + "github.com/golang/protobuf/ptypes/timestamp" + "gotest.tools/assert" +) + +func Test_TimestampValue(t *testing.T) { + t.Parallel() + + t.Run("valid", func(t *testing.T) { + ts := Timestamp{Timestamp: ×tamp.Timestamp{Seconds: 0, Nanos: 0}} + v, err := ts.Value() + assert.NilError(t, err) + assert.Equal(t, v, utcDate(1970, 1, 1)) + }) + t.Run("valid nil ts", func(t *testing.T) { + var ts *Timestamp + v, err := ts.Value() + assert.NilError(t, err) + assert.Equal(t, v, nil) + }) + t.Run("invalid ts", func(t *testing.T) { + ts := Timestamp{Timestamp: ×tamp.Timestamp{Seconds: maxValidSeconds, Nanos: 0}} + v, err := ts.Value() + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "timestamp: seconds:253402300800 after 10000-01-01") + assert.Equal(t, v, utcDate(10000, 1, 1)) + }) +} + +func Test_TimestampScan(t *testing.T) { + t.Parallel() + + t.Run("valid", func(t *testing.T) { + v := time.Unix(0, 0) + ts := Timestamp{} + err := ts.Scan(v) + assert.NilError(t, err) + assert.Check(t, reflect.DeepEqual(ts.Timestamp, ×tamp.Timestamp{Seconds: 0, Nanos: 0})) + }) + t.Run("valid default time", func(t *testing.T) { + var v time.Time + ts := Timestamp{} + err := ts.Scan(v) + assert.NilError(t, err) + assert.Check(t, reflect.DeepEqual(ts.Timestamp, ×tamp.Timestamp{Seconds: -62135596800, Nanos: 0})) + }) + t.Run("invalid type", func(t *testing.T) { + v := 1 + ts := Timestamp{} + err := ts.Scan(v) + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "Not a protobuf Timestamp") + }) + t.Run("invalid time", func(t *testing.T) { + v := time.Unix(maxValidSeconds, 0) + ts := Timestamp{} + err := ts.Scan(v) + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "error converting the timestamp: timestamp: seconds:253402300800 after 10000-01-01") + }) +} + +const maxValidSeconds = 253402300800 + +func utcDate(year, month, day int) time.Time { + return time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC) +} diff --git a/internal/oplog/store/scanners.go b/internal/oplog/store/scanners.go new file mode 100644 index 000000000..f438b1e6e --- /dev/null +++ b/internal/oplog/store/scanners.go @@ -0,0 +1,33 @@ +package store + +import ( + "database/sql/driver" + "errors" + "fmt" + "time" + + "github.com/golang/protobuf/ptypes" +) + +// Scan supports Timestamps for oplogs +func (ts *Timestamp) Scan(value interface{}) error { + switch t := value.(type) { + case time.Time: + var err error + ts.Timestamp, err = ptypes.TimestampProto(t) // google proto version + if err != nil { + return fmt.Errorf("error converting the timestamp: %w", err) + } + default: + return errors.New("Not a protobuf Timestamp") + } + return nil +} + +// Value supports Timestamps for oplogs +func (ts *Timestamp) Value() (driver.Value, error) { + if ts == nil { + return nil, nil + } + return ptypes.Timestamp(ts.Timestamp) +} diff --git a/internal/oplog/ticketer.go b/internal/oplog/ticketer.go new file mode 100644 index 000000000..c5f0253dd --- /dev/null +++ b/internal/oplog/ticketer.go @@ -0,0 +1,110 @@ +package oplog + +import ( + "errors" + "fmt" + + "github.com/hashicorp/watchtower/internal/oplog/store" + "github.com/jinzhu/gorm" +) + +var ( + ErrTicketNotFound = errors.New("ticket not found") + ErrTicketAlreadyRedeemed = errors.New("ticket already redeemed") + ErrTicketRedeeming = errors.New("error trying to redeem ticket") +) + +const DefaultAggregateName = "global" + +// Ticketer provides an interface to storage for Tickets, so you can easily substitute your own ticketer +type Ticketer interface { + // InitTicket initializes a ticket. You MUST initialize a ticket in its own transaction (and commit it), + // before you GetTicket in a later transaction to write to the oplog. InitTicket will check to see if + // the ticket has already been initialized before creating a new one. + InitTicket(aggregateName string) error + + // GetTicket returns a ticket for the specified name. You MUST GetTicket in the same transaction + // that you're using to write to the database tables. Names allow us to shard tickets around domain root names + GetTicket(aggregateName string) (*store.Ticket, error) + + Redeem(ticket *store.Ticket) error +} + +// GormTicketer uses a gorm DB connection for ticket storage +type GormTicketer struct { + tx *gorm.DB + withAggregateNames bool +} + +// NewGormTicketer creates a new ticketer that uses gorm for storage +func NewGormTicketer(tx *gorm.DB, opt ...Option) (*GormTicketer, error) { + if tx == nil { + return nil, errors.New("tx is nil") + } + opts := GetOpts(opt...) + enableAggregateNames := opts[optionWithAggregateNames].(bool) + return &GormTicketer{tx: tx, withAggregateNames: enableAggregateNames}, nil +} + +// GetTicket returns a ticket for the specified name. You MUST GetTicket in the same transaction +// that you're using to write to the database tables. Names allow us to shard tickets around domain root names +func (ticketer *GormTicketer) GetTicket(aggregateName string) (*store.Ticket, error) { + if aggregateName == "" { + return nil, errors.New("bad ticket name") + } + name := DefaultAggregateName + if ticketer.withAggregateNames { + name = aggregateName + } + ticket := store.Ticket{} + if err := ticketer.tx.First(&ticket, store.Ticket{Name: name}).Error; err != nil { + if gorm.IsRecordNotFoundError(err) { + return nil, ErrTicketNotFound + } + return nil, fmt.Errorf("error retreiving ticket from storage: %w", err) + } + return &ticket, nil +} + +// InitTicket initializes a ticket. You MUST initialize a ticket in its own transaction (and commit it), +// before you GetTicket in a later transaction to write to the oplog. InitTicket will check to see if +// the ticket has already been initialized before creating a new one. +func (ticketer *GormTicketer) InitTicket(aggregateName string) error { + // no existing ticket found, so let's initialize a new one + if aggregateName == "" { + return fmt.Errorf("bad ticket name") + } + // check to see if a ticket has already been initialized + existingTicket, err := ticketer.GetTicket(aggregateName) + if err == nil && existingTicket != nil { + return nil // found an existing intialized ticket without errors + } + if err != ErrTicketNotFound { + return fmt.Errorf("error retreiving ticket from storage: %w", err) + } + + name := DefaultAggregateName + if ticketer.withAggregateNames { + name = aggregateName + } + newTicket := store.Ticket{Name: name, Version: 1} + if err := ticketer.tx.Create(&newTicket).Error; err != nil { + return fmt.Errorf("error creating ticket in storage: %w", err) + } + return nil +} + +// Redeem will attempt to redeem the ticket. If the ticket version has already been used, then an error is returned +func (ticketer *GormTicketer) Redeem(t *store.Ticket) error { + if t == nil { + return errors.New("ticket is nil") + } + tx := ticketer.tx.Model(t).Where("version = ?", t.Version).Update("version", t.Version+1) + if tx.Error != nil { + return ErrTicketRedeeming + } + if tx.RowsAffected != 1 { + return ErrTicketAlreadyRedeemed + } + return nil +} diff --git a/internal/oplog/ticketer_gorm_test.go b/internal/oplog/ticketer_gorm_test.go new file mode 100644 index 000000000..0a1d5b2bc --- /dev/null +++ b/internal/oplog/ticketer_gorm_test.go @@ -0,0 +1,154 @@ +package oplog + +import ( + "testing" + + "github.com/hashicorp/go-uuid" + "gotest.tools/assert" +) + +// Test_NewGormTicketer provides unit tests for creating a Gorm ticketer +func Test_NewGormTicketer(t *testing.T) { + t.Parallel() + startTest() + cleanup, url := setup(t) + defer cleanup() + defer completeTest() // must come after the "defer cleanup()" + db, err := test_dbconn(url) + assert.NilError(t, err) + defer db.Close() + t.Run("valid", func(t *testing.T) { + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + assert.NilError(t, err) + assert.Assert(t, ticketer != nil) + }) + t.Run("bad db", func(t *testing.T) { + _, err := NewGormTicketer(nil, WithAggregateNames(true)) + assert.Equal(t, err.Error(), "tx is nil") + }) +} + +// Test_GetTicket provides unit tests for getting oplog.Tickets +func Test_GetTicket(t *testing.T) { + startTest() + t.Parallel() + cleanup, url := setup(t) + defer cleanup() + defer completeTest() // must come after the "defer cleanup()" + db, err := test_dbconn(url) + assert.NilError(t, err) + defer db.Close() + + ticketName, err := uuid.GenerateUUID() + assert.NilError(t, err) + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + assert.NilError(t, err) + + err = ticketer.InitTicket(ticketName) + assert.NilError(t, err) + + t.Run("valid", func(t *testing.T) { + ticket, err := ticketer.GetTicket(ticketName) + assert.NilError(t, err) + assert.Equal(t, ticket.Name, ticketName) + assert.Check(t, ticket.Version != 0) + }) + + t.Run("no name", func(t *testing.T) { + ticket, err := ticketer.GetTicket("") + assert.Equal(t, err.Error(), "bad ticket name") + assert.Check(t, ticket == nil) + }) +} + +// Test_InitTicket provides unit tests for initializing tickets +func Test_InitTicket(t *testing.T) { + startTest() + t.Parallel() + cleanup, url := setup(t) + defer cleanup() + defer completeTest() // must come after the "defer cleanup()" + db, err := test_dbconn(url) + assert.NilError(t, err) + defer db.Close() + + ticketName, err := uuid.GenerateUUID() + assert.NilError(t, err) + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + assert.NilError(t, err) + + t.Run("valid", func(t *testing.T) { + err = ticketer.InitTicket(ticketName) + assert.NilError(t, err) + }) + + t.Run("no name", func(t *testing.T) { + err = ticketer.InitTicket("") + assert.Equal(t, err.Error(), "bad ticket name") + }) +} + +// Test_Redeem provides unit tests for redeeming tickets +func Test_Redeem(t *testing.T) { + startTest() + t.Parallel() + cleanup, url := setup(t) + defer cleanup() + defer completeTest() // must come after the "defer cleanup()" + db, err := test_dbconn(url) + assert.NilError(t, err) + defer db.Close() + + ticketName, err := uuid.GenerateUUID() + assert.NilError(t, err) + ticketer, err := NewGormTicketer(db, WithAggregateNames(true)) + assert.NilError(t, err) + + // in it's own transaction, init the ticket + _ = ticketer.InitTicket(ticketName) + + t.Run("valid", func(t *testing.T) { + tx := db.Begin() + ticketer, err := NewGormTicketer(tx, WithAggregateNames(true)) + assert.NilError(t, err) + + ticket, err := ticketer.GetTicket(ticketName) + assert.NilError(t, err) + + err = ticketer.Redeem(ticket) + assert.NilError(t, err) + tx.Commit() + }) + + t.Run("nil ticket", func(t *testing.T) { + tx := db.Begin() + ticketer, err := NewGormTicketer(tx, WithAggregateNames(true)) + assert.NilError(t, err) + err = ticketer.Redeem(nil) + assert.Equal(t, err.Error(), "ticket is nil") + tx.Commit() + }) + + t.Run("detect two redemptions in separate concurrent transactions", func(t *testing.T) { + tx := db.Begin() + ticketer, err := NewGormTicketer(tx, WithAggregateNames(true)) + assert.NilError(t, err) + + ticket, err := ticketer.GetTicket(ticketName) + assert.NilError(t, err) + + secondTx := db.Begin() + secondTicketer, err := NewGormTicketer(secondTx, WithAggregateNames(true)) + assert.NilError(t, err) + secondTicket, err := secondTicketer.GetTicket(ticketName) + assert.NilError(t, err) + + err = ticketer.Redeem(ticket) + assert.NilError(t, err) + tx.Commit() + + err = secondTicketer.Redeem(secondTicket) + assert.Equal(t, err.Error(), "ticket already redeemed") + }) + +} diff --git a/internal/oplog/type_catalog.go b/internal/oplog/type_catalog.go new file mode 100644 index 000000000..ea61856fe --- /dev/null +++ b/internal/oplog/type_catalog.go @@ -0,0 +1,78 @@ +package oplog + +import ( + "errors" + fmt "fmt" + "reflect" +) + +// TypeCatalog is an abstraction for dealing with oplog data and their underlying types +type TypeCatalog map[string]reflect.Type + +// Type provides the ability to associate an interface with a Type.Name which +// will decouple the interface from it's reflection type string, so you can +// refactor the type name without breaking the catalog +type Type struct { + // Interface of the type + Interface interface{} + // Name for the interface + Name string +} + +// NewTypeCatalog creates a catalog with the types you pass in +func NewTypeCatalog(withTypes ...Type) (*TypeCatalog, error) { + reg := TypeCatalog{} + for _, t := range withTypes { + if t == (Type{}) { + return nil, errors.New("error type is {} (in NewTypeCatalog)") + + } + if err := reg.Set(t.Interface, t.Name); err != nil { + return nil, fmt.Errorf("error setting the type: %w (in NewTypeCatalog)", err) + } + } + return ®, nil +} + +// GetTypeName returns the interface's name from the catalog +func (t *TypeCatalog) GetTypeName(i interface{}) (string, error) { + if i == nil { + return "", errors.New("error interface parameter is nil for GetTypeName") + } + if reflect.ValueOf(i).Kind() != reflect.Ptr { + return "", errors.New("error interface parameter must to be a pointer for GetTypeName") + } + interfaceType := reflect.TypeOf(i) + for name, t := range *t { + if t == interfaceType { + return name, nil + } + } + return "", fmt.Errorf("error unknown name for interface: %T", i) +} + +// Set creates an entry in the catalog for the interface +func (t TypeCatalog) Set(i interface{}, typeName string) error { + if i == nil { + return errors.New("error interface parameter is nil for Set") + } + if reflect.ValueOf(i).Kind() != reflect.Ptr { + return errors.New("error interface parameter must to be a pointer for Set") + } + if typeName == "" { + return errors.New("typeName is an empty string for Set") + } + t[typeName] = reflect.TypeOf(i) + return nil +} + +// Get retrieves the interface via a name +func (t TypeCatalog) Get(typeName string) (interface{}, error) { + if typeName == "" { + return nil, errors.New("error typeName is empty string for Get") + } + if typ, ok := t[typeName]; ok { + return reflect.New(typ.Elem()).Elem().Addr().Interface(), nil + } + return nil, errors.New("error typeName is not found for Get") +} diff --git a/internal/oplog/type_catalog_test.go b/internal/oplog/type_catalog_test.go new file mode 100644 index 000000000..60ff09923 --- /dev/null +++ b/internal/oplog/type_catalog_test.go @@ -0,0 +1,187 @@ +package oplog + +import ( + "reflect" + "testing" + + "github.com/hashicorp/watchtower/internal/oplog/oplog_test" + "gotest.tools/assert" +) + +// Test_TypeCatalog provides basic red/green unit tests +func Test_TypeCatalog(t *testing.T) { + t.Parallel() + + types, err := NewTypeCatalog( + Type{new(oplog_test.TestUser), "user"}, + Type{new(oplog_test.TestCar), "car"}, + Type{new(oplog_test.TestRental), "rental"}, + ) + assert.NilError(t, err) + + name, err := types.GetTypeName(new(oplog_test.TestUser)) + assert.NilError(t, err) + assert.Assert(t, name == "user") + + _, err = types.GetTypeName(oplog_test.TestUser{}) + assert.Assert(t, err != nil) + + s := "string" + _, err = types.GetTypeName(&s) + assert.Assert(t, err != nil) + + _, err = types.Get("unknown") + assert.Assert(t, err != nil) + +} + +// Test_NewTypeCatalog provides unit tests for NewTypeCatalog +func Test_NewTypeCatalog(t *testing.T) { + t.Parallel() + + t.Run("valid", func(t *testing.T) { + types, err := NewTypeCatalog( + Type{new(oplog_test.TestUser), "user"}, + Type{new(oplog_test.TestCar), "car"}, + Type{new(oplog_test.TestRental), "rental"}, + ) + assert.NilError(t, err) + u, err := types.Get("user") + assert.NilError(t, err) + assert.Equal(t, reflect.TypeOf(u), reflect.TypeOf(new(oplog_test.TestUser))) + }) + t.Run("missing Type.Name", func(t *testing.T) { + types, err := NewTypeCatalog( + Type{new(oplog_test.TestUser), ""}, + ) + assert.Check(t, types == nil) + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "error setting the type: typeName is an empty string for Set (in NewTypeCatalog)") + }) + t.Run("missing Type.Interface", func(t *testing.T) { + types, err := NewTypeCatalog( + Type{nil, ""}, + ) + assert.Check(t, types == nil) + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "error type is {} (in NewTypeCatalog)") + }) + t.Run("empty Type", func(t *testing.T) { + types, err := NewTypeCatalog( + Type{}, + ) + assert.Check(t, types == nil) + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "error type is {} (in NewTypeCatalog)") + }) + +} + +// Test_GetTypeName provides unit tests for GetTypeName +func Test_GetTypeName(t *testing.T) { + t.Parallel() + + t.Run("valid", func(t *testing.T) { + types, err := NewTypeCatalog( + Type{new(oplog_test.TestUser), "user"}, + Type{new(oplog_test.TestCar), "car"}, + ) + assert.NilError(t, err) + + n, err := types.GetTypeName(new(oplog_test.TestUser)) + assert.NilError(t, err) + assert.Equal(t, n, "user") + }) + t.Run("bad name", func(t *testing.T) { + types, err := NewTypeCatalog( + Type{new(oplog_test.TestUser), "user"}, + ) + assert.NilError(t, err) + + n, err := types.GetTypeName(new(oplog_test.TestCar)) + assert.Check(t, err != nil) + assert.Equal(t, n, "") + assert.Equal(t, err.Error(), "error unknown name for interface: *oplog_test.TestCar") + }) + t.Run("nil interface", func(t *testing.T) { + types, err := NewTypeCatalog( + Type{new(oplog_test.TestUser), "user"}, + ) + assert.NilError(t, err) + + n, err := types.GetTypeName(nil) + assert.Check(t, err != nil) + assert.Equal(t, n, "") + assert.Equal(t, err.Error(), "error interface parameter is nil for GetTypeName") + }) +} + +// Test_Get provides unit tests for Get +func Test_Get(t *testing.T) { + t.Parallel() + + t.Run("valid", func(t *testing.T) { + types, err := NewTypeCatalog( + Type{new(oplog_test.TestUser), "user"}, + Type{new(oplog_test.TestCar), "car"}, + ) + assert.NilError(t, err) + + u, err := types.Get("user") + assert.NilError(t, err) + assert.Equal(t, reflect.TypeOf(u), reflect.TypeOf(new(oplog_test.TestUser))) + }) + t.Run("bad name", func(t *testing.T) { + types, err := NewTypeCatalog( + Type{new(oplog_test.TestUser), "user"}, + ) + assert.NilError(t, err) + + n, err := types.Get("car") + assert.Check(t, err != nil) + assert.Equal(t, n, nil) + assert.Equal(t, err.Error(), "error typeName is not found for Get") + }) + t.Run("bad typeName", func(t *testing.T) { + types, err := NewTypeCatalog( + Type{new(oplog_test.TestUser), "user"}, + ) + assert.NilError(t, err) + + n, err := types.Get("") + assert.Check(t, err != nil) + assert.Equal(t, n, nil) + assert.Equal(t, err.Error(), "error typeName is empty string for Get") + }) +} + +// Test_Set provides unit tests for Set +func Test_Set(t *testing.T) { + t.Parallel() + + t.Run("valid", func(t *testing.T) { + types, err := NewTypeCatalog() + assert.NilError(t, err) + u := new(oplog_test.TestUser) + err = types.Set(u, "user") + assert.NilError(t, err) + assert.Equal(t, reflect.TypeOf(u), reflect.TypeOf(new(oplog_test.TestUser))) + }) + t.Run("bad name", func(t *testing.T) { + types, err := NewTypeCatalog() + assert.NilError(t, err) + u := new(oplog_test.TestUser) + err = types.Set(u, "") + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "typeName is an empty string for Set") + assert.Assert(t, reflect.DeepEqual(types, &TypeCatalog{})) + }) + t.Run("bad interface", func(t *testing.T) { + types, err := NewTypeCatalog() + assert.NilError(t, err) + err = types.Set(nil, "") + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "error interface parameter is nil for Set") + assert.Assert(t, reflect.DeepEqual(types, &TypeCatalog{})) + }) +} diff --git a/internal/oplog/writer.go b/internal/oplog/writer.go new file mode 100644 index 000000000..c4940a476 --- /dev/null +++ b/internal/oplog/writer.go @@ -0,0 +1,101 @@ +package oplog + +import ( + "errors" + "fmt" + "reflect" + "strings" + + "github.com/jinzhu/gorm" +) + +// Writer interface for Entries +type Writer interface { + // Create the entry + Create(interface{}) error + + // Update the entry using the fieldMaskPaths, which are Paths from field_mask.proto + Update(entry interface{}, fieldMaskPaths []string) error + + // Delete the entry + Delete(interface{}) error +} + +// GormWriter uses a gorm DB connection for writing +type GormWriter struct { + Tx *gorm.DB +} + +// Create an object in storage +func (w *GormWriter) Create(i interface{}) error { + if w.Tx == nil { + return errors.New("create Tx is nil") + } + if i == nil { + return errors.New("create interface is nil") + } + if err := w.Tx.Create(i).Error; err != nil { + return fmt.Errorf("error creating: %w", err) + } + return nil +} + +// Update an object in storage, if there's a fieldMask then only the field_mask.proto paths are updated, otherwise +// we will send every field to the DB. +func (w *GormWriter) Update(i interface{}, fieldMaskPaths []string) error { + if w.Tx == nil { + return errors.New("update Tx is nil") + } + if i == nil { + return errors.New("update interface is nil") + } + if len(fieldMaskPaths) == 0 { + if err := w.Tx.Save(i).Error; err != nil { + return fmt.Errorf("error updating: %w", err) + } + } + updateFields := map[string]interface{}{} + + val := reflect.Indirect(reflect.ValueOf(i)) + structTyp := val.Type() + for _, field := range fieldMaskPaths { + for i := 0; i < structTyp.NumField(); i++ { + // support for an embedded a gorm type + if structTyp.Field(i).Type.Kind() == reflect.Struct { + embType := structTyp.Field(i).Type + // check if the embedded field is exported via CanInterface() + if val.Field(i).CanInterface() { + embVal := reflect.Indirect(reflect.ValueOf(val.Field(i).Interface())) + for embFieldNum := 0; embFieldNum < embType.NumField(); embFieldNum++ { + if strings.EqualFold(embType.Field(embFieldNum).Name, field) { + updateFields[field] = embVal.Field(embFieldNum).Interface() + } + } + continue + } + } + // it's not an embedded type, so check if the field name matches + if strings.EqualFold(structTyp.Field(i).Name, field) { + updateFields[field] = val.Field(i).Interface() + } + } + } + if err := w.Tx.Model(i).Updates(updateFields).Error; err != nil { + return fmt.Errorf("error updating: %w", err) + } + return nil +} + +// Deleting an object in storage +func (w *GormWriter) Delete(i interface{}) error { + if w.Tx == nil { + return errors.New("delete Tx is nil") + } + if i == nil { + return errors.New("delete interface is nil") + } + if err := w.Tx.Delete(i).Error; err != nil { + return fmt.Errorf("error deleting: %w", err) + } + return nil +} diff --git a/internal/oplog/writer_test.go b/internal/oplog/writer_test.go new file mode 100644 index 000000000..918e01c0c --- /dev/null +++ b/internal/oplog/writer_test.go @@ -0,0 +1,220 @@ +package oplog + +import ( + "testing" + + "github.com/hashicorp/go-uuid" + "github.com/hashicorp/watchtower/internal/oplog/oplog_test" + "github.com/jinzhu/gorm" + "gotest.tools/assert" +) + +// Test_GormWriterCreate provides unit tests for GormWriter Create +func Test_GormWriterCreate(t *testing.T) { + t.Parallel() + startTest() + cleanup, url := setup(t) + defer cleanup() + defer completeTest() // must come after the "defer cleanup()" + db, err := test_dbconn(url) + assert.NilError(t, err) + defer db.Close() + t.Run("valid", func(t *testing.T) { + tx := db.Begin() + defer tx.Rollback() + w := GormWriter{tx} + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + + user := oplog_test.TestUser{ + Name: "foo-" + id, + } + assert.NilError(t, w.Create(&user)) + + var foundUser oplog_test.TestUser + err = tx.Where("id = ?", user.Id).First(&foundUser).Error + assert.NilError(t, err) + assert.Equal(t, user.Name, foundUser.Name) + }) + t.Run("nil tx", func(t *testing.T) { + w := GormWriter{nil} + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + + user := oplog_test.TestUser{ + Name: "foo-" + id, + } + err = w.Create(&user) + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "create Tx is nil") + }) + t.Run("nil model", func(t *testing.T) { + tx := db.Begin() + defer tx.Rollback() + w := GormWriter{tx} + err = w.Create(nil) + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "create interface is nil") + }) +} + +// Test_GormWriterDelete provides unit tests for GormWriter Delete +func Test_GormWriterDelete(t *testing.T) { + t.Parallel() + startTest() + cleanup, url := setup(t) + defer cleanup() + defer completeTest() // must come after the "defer cleanup()" + db, err := test_dbconn(url) + assert.NilError(t, err) + defer db.Close() + t.Run("valid", func(t *testing.T) { + tx := db.Begin() + defer tx.Rollback() + w := GormWriter{tx} + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + + user := oplog_test.TestUser{ + Name: "foo-" + id, + } + assert.NilError(t, w.Create(&user)) + var foundUser oplog_test.TestUser + err = tx.Where("id = ?", user.Id).First(&foundUser).Error + assert.NilError(t, err) + + assert.NilError(t, w.Delete(&user)) + err = tx.Where("id = ?", user.Id).First(&foundUser).Error + assert.Check(t, err != nil) + assert.Equal(t, err, gorm.ErrRecordNotFound) + }) + t.Run("nil tx", func(t *testing.T) { + w := GormWriter{nil} + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + + user := oplog_test.TestUser{ + Name: "foo-" + id, + } + err = w.Delete(&user) + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "delete Tx is nil") + }) + t.Run("nil model", func(t *testing.T) { + tx := db.Begin() + defer tx.Rollback() + w := GormWriter{tx} + err = w.Delete(nil) + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "delete interface is nil") + }) +} + +// Test_GormWriterUpdate provides unit tests for GormWriter Update +func Test_GormWriterUpdate(t *testing.T) { + t.Parallel() + startTest() + cleanup, url := setup(t) + defer cleanup() + defer completeTest() // must come after the "defer cleanup()" + db, err := test_dbconn(url) + assert.NilError(t, err) + defer db.Close() + t.Run("valid no fieldmask", func(t *testing.T) { + tx := db.Begin() + defer tx.Rollback() + w := GormWriter{tx} + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + + user := oplog_test.TestUser{ + Name: "foo-" + id, + } + assert.NilError(t, w.Create(&user)) + var foundUser oplog_test.TestUser + err = tx.Where("id = ?", user.Id).First(&foundUser).Error + assert.NilError(t, err) + + foundUser.Name = foundUser.Name + "_updated" + assert.NilError(t, w.Update(&foundUser, nil)) + + var updatedUser oplog_test.TestUser + err = tx.Where("id = ?", user.Id).First(&updatedUser).Error + assert.NilError(t, err) + assert.Equal(t, foundUser.Name, updatedUser.Name) + }) + t.Run("valid with fieldmask", func(t *testing.T) { + tx := db.Begin() + defer tx.Rollback() + w := GormWriter{tx} + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + + user := oplog_test.TestUser{ + Name: "foo-" + id, + } + assert.NilError(t, w.Create(&user)) + var foundUser oplog_test.TestUser + err = tx.Where("id = ?", user.Id).First(&foundUser).Error + assert.NilError(t, err) + + foundUser.Name = foundUser.Name + "_updated" + assert.NilError(t, w.Update(&foundUser, []string{"Name"})) + + var updatedUser oplog_test.TestUser + err = tx.Where("id = ?", user.Id).First(&updatedUser).Error + assert.NilError(t, err) + assert.Equal(t, foundUser.Name, updatedUser.Name) + }) + t.Run("valid with incorrect fieldmask", func(t *testing.T) { + tx := db.Begin() + defer tx.Rollback() + w := GormWriter{tx} + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + + user := oplog_test.TestUser{ + Name: "foo-" + id, + } + assert.NilError(t, w.Create(&user)) + var foundUser oplog_test.TestUser + err = tx.Where("id = ?", user.Id).First(&foundUser).Error + assert.NilError(t, err) + + foundUser.Name = foundUser.Name + "_updated" + assert.NilError(t, w.Update(&foundUser, []string{"PhoneNumber"})) + + var updatedUser oplog_test.TestUser + err = tx.Where("id = ?", user.Id).First(&updatedUser).Error + assert.NilError(t, err) + assert.Check(t, updatedUser.Name != foundUser.Name+"_updated") + }) + t.Run("nil tx", func(t *testing.T) { + w := GormWriter{nil} + + id, err := uuid.GenerateUUID() + assert.NilError(t, err) + + user := oplog_test.TestUser{ + Name: "foo-" + id, + } + err = w.Update(&user, nil) + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "update Tx is nil") + }) + t.Run("nil model", func(t *testing.T) { + tx := db.Begin() + defer tx.Rollback() + w := GormWriter{tx} + err = w.Update(nil, nil) + assert.Check(t, err != nil) + assert.Equal(t, err.Error(), "update interface is nil") + }) +}