8000 feat: accounts data model by darkmatterpool · Pull Request #52 · formancehq/payments · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

feat: accounts data model #52

New issue

Have a question about this project? Sign 8000 up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 9, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
8000
Diff view
Diff view
8 changes: 1 addition & 7 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ require (
github.com/bombsimon/logrusr/v3 v3.0.0
github.com/davecgh/go-spew v1.1.1
github.com/gibson042/canonicaljson-go v1.0.3
github.com/google/uuid v1.3.0
github.com/gorilla/mux v1.8.0
github.com/iancoleman/strcase v0.2.0
github.com/numary/go-libs v1.0.1
Expand All @@ -26,7 +27,6 @@ require (
github.com/xdg-go/scram v1.1.1
go.mongodb.org/mongo-driver v1.10.1
go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.34.0
go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver/mongo/otelmongo v0.34.0
go.opentelemetry.io/otel v1.9.0
go.uber.org/dig v1.15.0
go.uber.org/fx v1.18.1
Expand All @@ -40,7 +40,6 @@ require (
github.com/ThreeDotsLabs/watermill-http v1.1.4 // indirect
github.com/ThreeDotsLabs/watermill-kafka/v2 v2.2.2 // indirect
github.com/ajg/form v1.5.1 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
Expand All @@ -58,13 +57,11 @@ require (
github.com/go-chi/render v1.0.2 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gofiber/fiber/v2 v2.38.1 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang/glog v1.0.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-cmp v0.5.8 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
Expand Down Expand Up @@ -97,9 +94,6 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.4.1 // indirect
github.com/uptrace/opentelemetry-go-extra/otelutil v0.1.15 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.40.0 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/stringprep v1.0.3 // indirect
github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
Expand Down
17 changes: 0 additions & 17 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
Expand Down Expand Up @@ -176,8 +174,6 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofiber/fiber/v2 v2.38.1 h1:GEQ/Yt3Wsf2a30iTqtLXlBYJZso0JXPovt/tmj5H9jU=
github.com/gofiber/fiber/v2 v2.38.1/go.mod h1:t0NlbaXzuGH7I+7M4paE848fNWInZ7mfxI/Er1fTth8=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
Expand Down Expand Up @@ -314,7 +310,6 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
Expand Down Expand Up @@ -478,12 +473,6 @@ github.com/uptrace/opentelemetry-go-extra/otelutil v0.1.15 h1:5eYO+onNB1mbdc3+uw
github.com/uptrace/opentelemetry-go-extra/otelutil v0.1.15/go.mod h1:6fGFPZDTcvHLxgWTFvf8hHWQrRO1tMXAFlxlqE+c650=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.40.0 h1:CRq/00MfruPGFLTQKY8b+8SfdK60TxNztjRMnH0t1Yc=
github.com/valyala/fasthttp v1.40.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
Expand Down Expand Up @@ -514,8 +503,6 @@ go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama
go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.34.0/go.mod h1:8cfNbNK5aJIRQnqOFGEALF17iZPDym2mmYP+ECIxi8M=
go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.34.0 h1:OkXMRbgldT4yZR7RwB4SFYTjYJGTXwPQVX69pYtTnc4=
go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.34.0/go.mod h1:zMu+r6aEorSQi8Ad0Y1fNrznm+VM8F10D2WlZp3HeFw=
go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver/mongo/otelmongo v0.34.0 h1:RBgtLcpCco6JtjXnoY3MstbUjDVWH7ECYn4WmzmlOBI=
go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver/mongo/otelmongo v0.34.0/go.mod h1:GIWhiaNpCoQJoD/R1q5GXySIwQIBQ+5pxMbuWhq3X94=
go.opentelemetry.io/otel v1.9.0 h1:8WZNQFIB2a71LnANS9JeyidJKKGOOremcUtb/OtHISw=
go.opentelemetry.io/otel v1.9.0/go.mod h1:np4EoPGzoPs3O67xUVNoPPcmSvsfOxNlNA4F4AC+0Eo=
go.opentelemetry.io/otel/exporters/jaeger v1.9.0 h1:gAEgEVGDWwFjcis9jJTOJqZNxDzoZfR12WNIxr7g9Ww=
Expand Down Expand Up @@ -561,7 +548,6 @@ golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 h1:vJ2V3lFLg+bBhgroYuRfyN583UzVveQmIXjc8T/y3to=
Expand Down Expand Up @@ -637,7 +623,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM=
golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c h1:JVAXQ10yGGVbSyoer5VILysz6YKjdNT2bsvlayjqhes=
Expand Down Expand Up @@ -721,9 +706,7 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
6 changes: 3 additions & 3 deletions internal/pkg/connectors/dummypay/task_ingest.go
6D40
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func taskIngest(config Config, descriptor TaskDescriptor, fs fs) task.Task {
}

// Ingest the payment into the system.
err = ingester.Ingest(ctx, ingestionPayload, struct{}{})
err = ingester.IngestPayments(ctx, ingestionPayload, struct{}{})
if err != nil {
return fmt.Errorf("failed to ingest file '%s': %w", descriptor.FileName, err)
}
Expand All @@ -39,7 +39,7 @@ func taskIngest(config Config, descriptor TaskDescriptor, fs fs) task.Task {
}
}

func parseIngestionPayload(config Config, descriptor TaskDescriptor, fs fs) (ingestion.Batch, error) {
func parseIngestionPayload(config Config, descriptor TaskDescriptor, fs fs) (ingestion.PaymentBatch, error) {
// Open the file.
file, err := fs.Open(filepath.Join(config.Directory, descriptor.FileName))
if err != nil {
Expand All @@ -56,7 +56,7 @@ func parseIngestionPayload(config Config, descriptor TaskDescriptor, fs fs) (ing
return nil, fmt.Errorf("failed to decode file '%s': %w", descriptor.FileName, err)
}

ingestionPayload := ingestion.Batch{ingestion.BatchElement{
ingestionPayload := ingestion.PaymentBatch{ingestion.PaymentBatchElement{
Referenced: payments.Referenced{
Reference: paymentElement.Reference,
Type: paymentElement.Type,
Expand Down
6 changes: 3 additions & 3 deletions internal/pkg/connectors/modulr/task_fetch_transactions.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,12 @@ func taskFetchTransactions(logger sharedlogging.Logger, client *client.Client, a
return err
}

batch := ingestion.Batch{}
batch := ingestion.PaymentBatch{}

for _, transaction := range transactions {
logger.Info(transaction)

batchElement := ingestion.BatchElement{
batchElement := ingestion.PaymentBatchElement{
Referenced: payments.Referenced{
Reference: transaction.ID,
Type: matchTransactionType(transaction.Type),
Expand All @@ -48,7 +48,7 @@ func taskFetchTransactions(logger sharedlogging.Logger, client *client.Client, a
batch = append(batch, batchElement)
}

return ingester.Ingest(ctx, batch, struct{}{})
return ingester.IngestPayments(ctx, batch, struct{}{})
}
}

Expand Down
4 changes: 2 additions & 2 deletions internal/pkg/connectors/stripe/task_connected_account.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
func ingestBatch(ctx context.Context, logger sharedlogging.Logger, ingester ingestion.Ingester,
bts []*stripe.BalanceTransaction, commitState TimelineState, tail bool,
) error {
batch := ingestion.Batch{}
batch := ingestion.PaymentBatch{}

for _, bt := range bts {
batchElement, handled := CreateBatchElement(bt, !tail)
Expand All @@ -37,7 +37,7 @@ func ingestBatch(ctx context.Context, logger sharedlogging.Logger, ingester inge
"state": commitState,
}).Debugf("updating state")

err := ingester.Ingest(ctx, batch, commitState)
err := ingester.IngestPayments(ctx, batch, commitState)
if err != nil {
return err
}
Expand Down
12 changes: 6 additions & 6 deletions internal/pkg/connectors/stripe/translate.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func currencies() map[string]currency {
}
}

func CreateBatchElement(balanceTransaction *stripe.BalanceTransaction, forward bool) (ingestion.BatchElement, bool) {
func CreateBatchElement(balanceTransaction *stripe.BalanceTransaction, forward bool) (ingestion.PaymentBatchElement, bool) {
var (
reference payments.Referenced
paymentData *payments.Data
Expand All @@ -145,11 +145,11 @@ func CreateBatchElement(balanceTransaction *stripe.BalanceTransaction, forward b
}()

if balanceTransaction.Source == nil {
return ingestion.BatchElement{}, false
return ingestion.PaymentBatchElement{}, false
}

if balanceTransaction.Source.Payout == nil {
return ingestion.BatchElement{}, false
return ingestion.PaymentBatchElement{}, false
}

formatAsset := func(cur stripe.Currency) string {
Expand Down Expand Up @@ -290,12 +290,12 @@ func CreateBatchElement(balanceTransaction *stripe.BalanceTransaction, forward b
Raw: balanceTransaction,
}
case "stripe_fee", "network_cost":
return ingestion.BatchElement{}, true
return ingestion.PaymentBatchElement{}, true
default:
return ingestion.BatchElement{}, false
return ingestion.PaymentBatchElement{}, false
}

return ingestion.BatchElement{
return ingestion.PaymentBatchElement{
Referenced: reference,
Payment: paymentData,
Adjustment: adjustment,
Expand Down
42 changes: 38 additions & 4 deletions internal/pkg/connectors/wise/task_fetch_transfers.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,15 @@ func taskFetchTransfers(logger sharedlogging.Logger, client *client, profileID u
return err
}

batch := ingestion.Batch{}
var (
accountBatch ingestion.AccountBatch
paymentBatch ingestion.PaymentBatch
)

for _, transfer := range transfers {
logger.Info(transfer)

batchElement := ingestion.BatchElement{
batchElement := ingestion.PaymentBatchElement{
Referenced: payments.Referenced{
Reference: fmt.Sprintf("%d", transfer.ID),
Type: payments.TypeTransfer,
Expand All @@ -42,10 +45,41 @@ func taskFetchTransfers(logger sharedlogging.Logger, client *client, profileID u
},
}

batch = append(batch, batchElement)
if transfer.SourceAccount != 0 {
ref := fmt.Sprintf("%d", transfer.SourceAccount)

accountBatch = append(accountBatch,
ingestion.AccountBatchElement{
Reference: ref,
Type: payments.AccountTypeSource,
},
)

batchElement.Referenced.Accounts = append(batchElement.Referenced.Accounts, ref)
}

if transfer.TargetAccount != 0 {
ref := fmt.Sprintf("%d", transfer.TargetAccount)

accountBatch = append(accountBatch,
ingestion.AccountBatchElement{
Reference: ref,
Type: payments.AccountTypeTarget,
},
)

batchElement.Referenced.Accounts = append(batchElement.Referenced.Accounts, ref)
}

paymentBatch = append(paymentBatch, batchElement)
}

err = ingester.IngestAccounts(ctx, accountBatch)
if err != nil {
return err
}

return ingester.Ingest(ctx, batch, struct{}{})
return ingester.IngestPayments(ctx, paymentBatch, struct{}{})
}
}

Expand Down
Loading
0