feat(#18): import de extratos OFX e CSV — parser, dedup, preview e confirmação
Parser OFX tolerante ao formato SGML de bancos brasileiros (sem closing tags, BOM, timezone suffix). Parser CSV com separador de campo/decimal configurável. Dedup por FITID (OFX) ou date+amount+desc normalizado. Fluxo: POST /preview → revisão na UI → POST /confirm salva e loga. Migration 002 adiciona external_id com unique index. Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
Claude Sonnet 4.6
@@ -52,6 +52,10 @@ func main() {
|
||||
categorySvc := service.NewCategoryService(categoryRepo)
|
||||
categoryHandler := handler.NewCategoryHandler(categorySvc)
|
||||
|
||||
transactionRepo := repository.NewTransactionRepository(pool)
|
||||
importSvc := service.NewImportService(transactionRepo)
|
||||
importHandler := handler.NewImportHandler(importSvc)
|
||||
|
||||
r.Get("/health", handler.Health)
|
||||
|
||||
r.Route("/api", func(r chi.Router) {
|
||||
@@ -59,6 +63,9 @@ func main() {
|
||||
r.Post("/categories", categoryHandler.Create)
|
||||
r.Put("/categories/{id}", categoryHandler.Update)
|
||||
r.Delete("/categories/{id}", categoryHandler.Delete)
|
||||
|
||||
r.Post("/imports/preview", importHandler.Preview)
|
||||
r.Post("/imports/confirm", importHandler.Confirm)
|
||||
})
|
||||
|
||||
// Serve Vue SPA — non-API routes fall through to index.html
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"financeiro-carvalho/internal/model"
|
||||
"financeiro-carvalho/internal/service"
|
||||
)
|
||||
|
||||
const maxUploadSize = 10 << 20 // 10 MB
|
||||
|
||||
type ImportHandler struct {
|
||||
svc *service.ImportService
|
||||
}
|
||||
|
||||
func NewImportHandler(svc *service.ImportService) *ImportHandler {
|
||||
return &ImportHandler{svc: svc}
|
||||
}
|
||||
|
||||
// Preview parses the uploaded file and returns rows with duplicate flags.
|
||||
// POST /api/imports/preview multipart: file + optional csv_mapping JSON field
|
||||
func (h *ImportHandler) Preview(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxUploadSize)
|
||||
if err := r.ParseMultipartForm(maxUploadSize); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "file too large or invalid form")
|
||||
return
|
||||
}
|
||||
|
||||
file, header, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadRequest, "field 'file' is required")
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var csvMapping *model.CSVMapping
|
||||
if raw := r.FormValue("csv_mapping"); raw != "" {
|
||||
var m model.CSVMapping
|
||||
if err := json.Unmarshal([]byte(raw), &m); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "invalid csv_mapping JSON")
|
||||
return
|
||||
}
|
||||
csvMapping = &m
|
||||
}
|
||||
|
||||
rows, parseErrs, err := h.svc.Preview(r.Context(), header.Filename, file, csvMapping)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]any{
|
||||
"rows": rows,
|
||||
"parse_errors": parseErrs,
|
||||
})
|
||||
}
|
||||
|
||||
// Confirm saves the rows provided in the request body (after user review).
|
||||
// POST /api/imports/confirm
|
||||
func (h *ImportHandler) Confirm(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Filename string `json:"filename"`
|
||||
Rows []model.ImportRow `json:"rows"`
|
||||
ParseErrCount int `json:"parse_error_count"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "invalid JSON")
|
||||
return
|
||||
}
|
||||
if body.Filename == "" {
|
||||
respondError(w, http.StatusBadRequest, "filename is required")
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.svc.Confirm(r.Context(), body.Filename, body.Rows, body.ParseErrCount)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "failed to save transactions")
|
||||
return
|
||||
}
|
||||
respondJSON(w, http.StatusOK, result)
|
||||
}
|
||||
@@ -9,11 +9,22 @@ import (
|
||||
)
|
||||
|
||||
//go:embed sql/001_initial.sql
|
||||
var initial string
|
||||
var m001 string
|
||||
|
||||
//go:embed sql/002_add_external_id.sql
|
||||
var m002 string
|
||||
|
||||
func Run(ctx context.Context, pool *pgxpool.Pool) error {
|
||||
if _, err := pool.Exec(ctx, initial); err != nil {
|
||||
return fmt.Errorf("001_initial: %w", err)
|
||||
for i, sql := range []struct {
|
||||
version int
|
||||
sql string
|
||||
}{
|
||||
{1, m001},
|
||||
{2, m002},
|
||||
} {
|
||||
if _, err := pool.Exec(ctx, sql.sql); err != nil {
|
||||
return fmt.Errorf("migration %03d: %w", i+1, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version BIGINT PRIMARY KEY,
|
||||
applied_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS categories (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
color VARCHAR(7) NOT NULL DEFAULT '#6B7280',
|
||||
is_default BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS transactions (
|
||||
id SERIAL PRIMARY KEY,
|
||||
date DATE NOT NULL,
|
||||
amount NUMERIC(12, 2) NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
type VARCHAR(10) NOT NULL CHECK (type IN ('income', 'expense')),
|
||||
source VARCHAR(10) NOT NULL DEFAULT 'manual' CHECK (source IN ('manual', 'import')),
|
||||
category_id INTEGER REFERENCES categories (id) ON DELETE SET NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS recurring_expenses (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
expected_amount NUMERIC(12, 2) NOT NULL,
|
||||
day_of_month INTEGER NOT NULL CHECK (day_of_month BETWEEN 1 AND 31),
|
||||
category_id INTEGER REFERENCES categories (id) ON DELETE SET NULL,
|
||||
active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS import_logs (
|
||||
id SERIAL PRIMARY KEY,
|
||||
filename VARCHAR(255) NOT NULL,
|
||||
format VARCHAR(10) NOT NULL CHECK (format IN ('ofx', 'csv')),
|
||||
imported_count INTEGER NOT NULL DEFAULT 0,
|
||||
duplicate_count INTEGER NOT NULL DEFAULT 0,
|
||||
error_count INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
INSERT INTO categories (name, color, is_default)
|
||||
VALUES
|
||||
('Saúde / Insulina', '#EF4444', TRUE),
|
||||
('Veleiro', '#3B82F6', TRUE),
|
||||
('Jogos de Tabuleiro', '#8B5CF6', TRUE),
|
||||
('Alimentação', '#F59E0B', TRUE),
|
||||
('Lazer', '#10B981', TRUE),
|
||||
('Freelance', '#6366F1', TRUE),
|
||||
('Outros', '#6B7280', TRUE)
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO schema_migrations (version) VALUES (1) ON CONFLICT DO NOTHING;
|
||||
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE transactions ADD COLUMN IF NOT EXISTS external_id TEXT;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS transactions_external_id_idx ON transactions (external_id) WHERE external_id IS NOT NULL;
|
||||
|
||||
INSERT INTO schema_migrations (version) VALUES (2) ON CONFLICT DO NOTHING;
|
||||
@@ -0,0 +1,43 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
type Transaction struct {
|
||||
ID int `json:"id"`
|
||||
Date string `json:"date"` // YYYY-MM-DD
|
||||
Amount float64 `json:"amount"`
|
||||
Description string `json:"description"`
|
||||
Type string `json:"type"` // income | expense
|
||||
Source string `json:"source"` // manual | import
|
||||
CategoryID *int `json:"category_id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// ImportRow is a parsed-but-not-yet-saved transaction from a file.
|
||||
type ImportRow struct {
|
||||
Date string `json:"date"`
|
||||
Amount float64 `json:"amount"`
|
||||
Description string `json:"description"`
|
||||
Type string `json:"type"`
|
||||
ExternalID string `json:"external_id,omitempty"` // FITID from OFX
|
||||
IsDuplicate bool `json:"is_duplicate"`
|
||||
}
|
||||
|
||||
// ImportResult is the response after confirming an import.
|
||||
type ImportResult struct {
|
||||
Imported int `json:"imported"`
|
||||
Duplicates int `json:"duplicates"`
|
||||
Errors int `json:"errors"`
|
||||
}
|
||||
|
||||
// CSVMapping tells the parser which CSV column index holds each field.
|
||||
type CSVMapping struct {
|
||||
DateColumn int `json:"date_column"`
|
||||
AmountColumn int `json:"amount_column"`
|
||||
DescriptionColumn int `json:"description_column"`
|
||||
DateFormat string `json:"date_format"` // Go time format, e.g. "02/01/2006"
|
||||
HasHeader bool `json:"has_header"`
|
||||
DecimalSeparator string `json:"decimal_separator"` // "." or ","
|
||||
FieldSeparator string `json:"field_separator"` // "," or ";"
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"financeiro-carvalho/internal/model"
|
||||
)
|
||||
|
||||
func ParseCSV(r io.Reader, m model.CSVMapping) ([]model.ImportRow, []string) {
|
||||
if m.DateFormat == "" {
|
||||
m.DateFormat = "02/01/2006"
|
||||
}
|
||||
if m.DecimalSeparator == "" {
|
||||
m.DecimalSeparator = ","
|
||||
}
|
||||
fieldSep := ','
|
||||
if m.FieldSeparator == ";" {
|
||||
fieldSep = ';'
|
||||
}
|
||||
|
||||
reader := csv.NewReader(r)
|
||||
reader.Comma = fieldSep
|
||||
reader.LazyQuotes = true
|
||||
reader.TrimLeadingSpace = true
|
||||
|
||||
var rows []model.ImportRow
|
||||
var errs []string
|
||||
lineNum := 0
|
||||
|
||||
for {
|
||||
record, err := reader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
lineNum++
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Sprintf("linha %d: %v", lineNum, err))
|
||||
continue
|
||||
}
|
||||
if lineNum == 1 && m.HasHeader {
|
||||
continue
|
||||
}
|
||||
|
||||
maxCol := max3(m.DateColumn, m.AmountColumn, m.DescriptionColumn)
|
||||
if len(record) <= maxCol {
|
||||
errs = append(errs, fmt.Sprintf("linha %d: colunas insuficientes (%d)", lineNum, len(record)))
|
||||
continue
|
||||
}
|
||||
|
||||
dateStr := strings.TrimSpace(record[m.DateColumn])
|
||||
amtStr := strings.TrimSpace(record[m.AmountColumn])
|
||||
desc := cleanDescription(record[m.DescriptionColumn])
|
||||
|
||||
t, err := time.Parse(m.DateFormat, dateStr)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Sprintf("linha %d: data inválida %q", lineNum, dateStr))
|
||||
continue
|
||||
}
|
||||
|
||||
if m.DecimalSeparator == "," {
|
||||
amtStr = strings.ReplaceAll(amtStr, ".", "")
|
||||
amtStr = strings.ReplaceAll(amtStr, ",", ".")
|
||||
}
|
||||
amt, err := strconv.ParseFloat(amtStr, 64)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Sprintf("linha %d: valor inválido %q", lineNum, amtStr))
|
||||
continue
|
||||
}
|
||||
|
||||
txType := "expense"
|
||||
if amt > 0 {
|
||||
txType = "income"
|
||||
}
|
||||
|
||||
rows = append(rows, model.ImportRow{
|
||||
Date: t.Format("2006-01-02"),
|
||||
Amount: math.Abs(amt),
|
||||
Description: desc,
|
||||
Type: txType,
|
||||
})
|
||||
}
|
||||
return rows, errs
|
||||
}
|
||||
|
||||
func max3(a, b, c int) int {
|
||||
if b > a {
|
||||
a = b
|
||||
}
|
||||
if c > a {
|
||||
a = c
|
||||
}
|
||||
return a
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"financeiro-carvalho/internal/model"
|
||||
)
|
||||
|
||||
// ParseOFX parses OFX SGML files as exported by Brazilian banks.
|
||||
// It is tolerant of the non-strict SGML format (no closing tags, BOM, etc.).
|
||||
func ParseOFX(r io.Reader) ([]model.ImportRow, error) {
|
||||
scanner := bufio.NewScanner(r)
|
||||
scanner.Buffer(make([]byte, 1024*1024), 1024*1024)
|
||||
|
||||
var rows []model.ImportRow
|
||||
var cur *model.ImportRow
|
||||
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
line = strings.TrimPrefix(line, "\xef\xbb\xbf") // strip UTF-8 BOM if present
|
||||
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
tag, value := splitOFXLine(line)
|
||||
|
||||
switch tag {
|
||||
case "STMTTRN":
|
||||
if cur != nil {
|
||||
rows = append(rows, *cur)
|
||||
}
|
||||
cur = &model.ImportRow{}
|
||||
case "/STMTTRN":
|
||||
if cur != nil {
|
||||
rows = append(rows, *cur)
|
||||
cur = nil
|
||||
}
|
||||
case "DTPOSTED":
|
||||
if cur != nil {
|
||||
cur.Date = parseOFXDate(value)
|
||||
}
|
||||
case "TRNAMT":
|
||||
if cur != nil {
|
||||
amt, _ := strconv.ParseFloat(strings.ReplaceAll(value, ",", "."), 64)
|
||||
cur.Amount = math.Abs(amt)
|
||||
if amt < 0 {
|
||||
cur.Type = "expense"
|
||||
} else {
|
||||
cur.Type = "income"
|
||||
}
|
||||
}
|
||||
case "FITID":
|
||||
if cur != nil {
|
||||
cur.ExternalID = value
|
||||
}
|
||||
case "MEMO":
|
||||
if cur != nil && cur.Description == "" {
|
||||
cur.Description = cleanDescription(value)
|
||||
}
|
||||
case "NAME":
|
||||
if cur != nil {
|
||||
cur.Description = cleanDescription(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
if cur != nil {
|
||||
rows = append(rows, *cur)
|
||||
}
|
||||
return rows, scanner.Err()
|
||||
}
|
||||
|
||||
// splitOFXLine handles both "<TAG>value" and "TAG:value" formats.
|
||||
func splitOFXLine(line string) (tag, value string) {
|
||||
if strings.HasPrefix(line, "<") {
|
||||
end := strings.Index(line, ">")
|
||||
if end < 0 {
|
||||
return "", ""
|
||||
}
|
||||
tag = strings.ToUpper(line[1:end])
|
||||
value = strings.TrimSpace(line[end+1:])
|
||||
return
|
||||
}
|
||||
// header lines like "OFXHEADER:100" — ignore
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// parseOFXDate handles formats: 20060102, 20060102120000, 20060102120000[-3:BRT]
|
||||
func parseOFXDate(s string) string {
|
||||
s = strings.SplitN(s, "[", 2)[0] // strip timezone suffix
|
||||
s = strings.TrimSpace(s)
|
||||
formats := []string{"20060102150405", "20060102"}
|
||||
for _, f := range formats {
|
||||
if len(s) >= len(f) {
|
||||
t, err := time.Parse(f, s[:len(f)])
|
||||
if err == nil {
|
||||
return t.Format("2006-01-02")
|
||||
}
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func cleanDescription(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
// remove duplicate whitespace
|
||||
fields := strings.Fields(s)
|
||||
return strings.Join(fields, " ")
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package parser_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"financeiro-carvalho/internal/model"
|
||||
"financeiro-carvalho/internal/parser"
|
||||
)
|
||||
|
||||
const sampleOFX = `OFXHEADER:100
|
||||
DATA:OFXSGML
|
||||
VERSION:150
|
||||
|
||||
<OFX>
|
||||
<BANKMSGSRSV1>
|
||||
<STMTTRNRS>
|
||||
<STMTRS>
|
||||
<BANKTRANLIST>
|
||||
<STMTTRN>
|
||||
<TRNTYPE>DEBIT
|
||||
<DTPOSTED>20240315120000[-3:BRT]
|
||||
<TRNAMT>-150.90
|
||||
<FITID>2024031500001
|
||||
<MEMO>PAGTO BOLETO ENERGIA
|
||||
</STMTTRN>
|
||||
<STMTTRN>
|
||||
<TRNTYPE>CREDIT
|
||||
<DTPOSTED>20240320
|
||||
<TRNAMT>5000.00
|
||||
<FITID>2024032000001
|
||||
<MEMO>SALARIO EMPRESA
|
||||
</STMTTRN>
|
||||
</BANKTRANLIST>
|
||||
`
|
||||
|
||||
func TestParseOFX_BasicFields(t *testing.T) {
|
||||
rows, err := parser.ParseOFX(strings.NewReader(sampleOFX))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("expected 2 rows, got %d", len(rows))
|
||||
}
|
||||
|
||||
debit := rows[0]
|
||||
if debit.Date != "2024-03-15" {
|
||||
t.Errorf("date: want 2024-03-15, got %q", debit.Date)
|
||||
}
|
||||
if debit.Amount != 150.90 {
|
||||
t.Errorf("amount: want 150.90, got %v", debit.Amount)
|
||||
}
|
||||
if debit.Type != "expense" {
|
||||
t.Errorf("type: want expense, got %q", debit.Type)
|
||||
}
|
||||
if debit.ExternalID != "2024031500001" {
|
||||
t.Errorf("external_id: got %q", debit.ExternalID)
|
||||
}
|
||||
if debit.Description != "PAGTO BOLETO ENERGIA" {
|
||||
t.Errorf("desc: got %q", debit.Description)
|
||||
}
|
||||
|
||||
credit := rows[1]
|
||||
if credit.Type != "income" {
|
||||
t.Errorf("type: want income, got %q", credit.Type)
|
||||
}
|
||||
if credit.Amount != 5000.00 {
|
||||
t.Errorf("amount: want 5000.00, got %v", credit.Amount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseOFX_EmptyFile(t *testing.T) {
|
||||
rows, err := parser.ParseOFX(strings.NewReader(""))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 0 {
|
||||
t.Fatalf("expected 0 rows, got %d", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCSV_Semicolon(t *testing.T) {
|
||||
// Brazilian bank CSV: semicolon separator, comma as decimal
|
||||
data := "15/03/2024;-150,90;SUPERMERCADO\n20/03/2024;5000,00;SALARIO\n"
|
||||
m := model.CSVMapping{
|
||||
DateColumn: 0,
|
||||
AmountColumn: 1,
|
||||
DescriptionColumn: 2,
|
||||
DateFormat: "02/01/2006",
|
||||
HasHeader: false,
|
||||
DecimalSeparator: ",",
|
||||
FieldSeparator: ";",
|
||||
}
|
||||
rows, errs := parser.ParseCSV(strings.NewReader(data), m)
|
||||
if len(errs) != 0 {
|
||||
t.Fatalf("unexpected errors: %v", errs)
|
||||
}
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("expected 2 rows, got %d", len(rows))
|
||||
}
|
||||
if rows[0].Date != "2024-03-15" {
|
||||
t.Errorf("date: got %q", rows[0].Date)
|
||||
}
|
||||
if rows[0].Amount != 150.90 {
|
||||
t.Errorf("amount: want 150.90, got %v", rows[0].Amount)
|
||||
}
|
||||
if rows[0].Type != "expense" {
|
||||
t.Errorf("type: got %q", rows[0].Type)
|
||||
}
|
||||
if rows[1].Type != "income" {
|
||||
t.Errorf("income type: got %q", rows[1].Type)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCSV_WithHeader(t *testing.T) {
|
||||
data := "Data;Valor;Descricao\n15/03/2024;-50,00;FARMACIA\n"
|
||||
m := model.CSVMapping{
|
||||
DateColumn: 0,
|
||||
AmountColumn: 1,
|
||||
DescriptionColumn: 2,
|
||||
DateFormat: "02/01/2006",
|
||||
HasHeader: true,
|
||||
DecimalSeparator: ",",
|
||||
FieldSeparator: ";",
|
||||
}
|
||||
rows, errs := parser.ParseCSV(strings.NewReader(data), m)
|
||||
if len(errs) != 0 {
|
||||
t.Fatalf("errors: %v", errs)
|
||||
}
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("expected 1 row (header skipped), got %d", len(rows))
|
||||
}
|
||||
if rows[0].Description != "FARMACIA" {
|
||||
t.Errorf("desc: got %q", rows[0].Description)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"financeiro-carvalho/internal/model"
|
||||
)
|
||||
|
||||
type TransactionRepository interface {
|
||||
IsDuplicate(ctx context.Context, date, description string, amount float64) (bool, error)
|
||||
IsExternalIDKnown(ctx context.Context, externalID string) (bool, error)
|
||||
BulkInsert(ctx context.Context, rows []model.ImportRow) (int, error)
|
||||
SaveImportLog(ctx context.Context, filename, format string, imported, duplicates, errors int) error
|
||||
}
|
||||
|
||||
type transactionRepo struct{ db *pgxpool.Pool }
|
||||
|
||||
func NewTransactionRepository(db *pgxpool.Pool) TransactionRepository {
|
||||
return &transactionRepo{db: db}
|
||||
}
|
||||
|
||||
// normalizeDesc lowercases and collapses whitespace for fuzzy dedup.
|
||||
func normalizeDesc(s string) string {
|
||||
return strings.ToLower(strings.Join(strings.Fields(s), " "))
|
||||
}
|
||||
|
||||
func (r *transactionRepo) IsDuplicate(ctx context.Context, date, description string, amount float64) (bool, error) {
|
||||
var count int
|
||||
err := r.db.QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM transactions
|
||||
WHERE date = $1 AND amount = $2 AND LOWER(description) = $3`,
|
||||
date, amount, normalizeDesc(description)).Scan(&count)
|
||||
return count > 0, err
|
||||
}
|
||||
|
||||
func (r *transactionRepo) IsExternalIDKnown(ctx context.Context, externalID string) (bool, error) {
|
||||
if externalID == "" {
|
||||
return false, nil
|
||||
}
|
||||
var count int
|
||||
err := r.db.QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM transactions WHERE external_id = $1`, externalID).Scan(&count)
|
||||
return count > 0, err
|
||||
}
|
||||
|
||||
func (r *transactionRepo) BulkInsert(ctx context.Context, rows []model.ImportRow) (int, error) {
|
||||
tx, err := r.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
count := 0
|
||||
for _, row := range rows {
|
||||
if row.IsDuplicate {
|
||||
continue
|
||||
}
|
||||
var extID *string
|
||||
if row.ExternalID != "" {
|
||||
extID = &row.ExternalID
|
||||
}
|
||||
_, err := tx.Exec(ctx, `
|
||||
INSERT INTO transactions (date, amount, description, type, source, external_id)
|
||||
VALUES ($1, $2, $3, $4, 'import', $5)`,
|
||||
row.Date, row.Amount, row.Description, row.Type, extID)
|
||||
if err != nil {
|
||||
return count, err
|
||||
}
|
||||
count++
|
||||
}
|
||||
return count, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (r *transactionRepo) SaveImportLog(ctx context.Context, filename, format string, imported, duplicates, errors int) error {
|
||||
_, err := r.db.Exec(ctx, `
|
||||
INSERT INTO import_logs (filename, format, imported_count, duplicate_count, error_count)
|
||||
VALUES ($1, $2, $3, $4, $5)`,
|
||||
filename, format, imported, duplicates, errors)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"financeiro-carvalho/internal/model"
|
||||
"financeiro-carvalho/internal/parser"
|
||||
"financeiro-carvalho/internal/repository"
|
||||
)
|
||||
|
||||
type ImportService struct {
|
||||
repo repository.TransactionRepository
|
||||
}
|
||||
|
||||
func NewImportService(repo repository.TransactionRepository) *ImportService {
|
||||
return &ImportService{repo: repo}
|
||||
}
|
||||
|
||||
// Preview parses the file and marks duplicates without saving anything.
|
||||
func (s *ImportService) Preview(ctx context.Context, filename string, r io.Reader, csvMapping *model.CSVMapping) ([]model.ImportRow, []string, error) {
|
||||
rows, parseErrs, err := s.parse(filename, r, csvMapping)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
for i := range rows {
|
||||
dup, err := s.isDuplicate(ctx, &rows[i])
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("dedup check: %w", err)
|
||||
}
|
||||
rows[i].IsDuplicate = dup
|
||||
}
|
||||
return rows, parseErrs, nil
|
||||
}
|
||||
|
||||
// Confirm saves the non-duplicate rows and logs the operation.
|
||||
func (s *ImportService) Confirm(ctx context.Context, filename string, rows []model.ImportRow, parseErrCount int) (model.ImportResult, error) {
|
||||
imported, err := s.repo.BulkInsert(ctx, rows)
|
||||
if err != nil {
|
||||
return model.ImportResult{}, err
|
||||
}
|
||||
|
||||
duplicates := 0
|
||||
for _, r := range rows {
|
||||
if r.IsDuplicate {
|
||||
duplicates++
|
||||
}
|
||||
}
|
||||
|
||||
format := strings.TrimPrefix(strings.ToLower(filepath.Ext(filename)), ".")
|
||||
if format != "ofx" && format != "csv" {
|
||||
format = "csv"
|
||||
}
|
||||
|
||||
_ = s.repo.SaveImportLog(ctx, filename, format, imported, duplicates, parseErrCount)
|
||||
return model.ImportResult{
|
||||
Imported: imported,
|
||||
Duplicates: duplicates,
|
||||
Errors: parseErrCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ImportService) parse(filename string, r io.Reader, csvMapping *model.CSVMapping) ([]model.ImportRow, []string, error) {
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
switch ext {
|
||||
case ".ofx":
|
||||
rows, err := parser.ParseOFX(r)
|
||||
return rows, nil, err
|
||||
case ".csv":
|
||||
m := model.CSVMapping{
|
||||
DateColumn: 0,
|
||||
AmountColumn: 1,
|
||||
DescriptionColumn: 2,
|
||||
DateFormat: "02/01/2006",
|
||||
HasHeader: true,
|
||||
DecimalSeparator: ",",
|
||||
FieldSeparator: ";",
|
||||
}
|
||||
if csvMapping != nil {
|
||||
m = *csvMapping
|
||||
}
|
||||
rows, errs := parser.ParseCSV(r, m)
|
||||
return rows, errs, nil
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unsupported file format: %q (use .ofx or .csv)", ext)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ImportService) isDuplicate(ctx context.Context, row *model.ImportRow) (bool, error) {
|
||||
if row.ExternalID != "" {
|
||||
return s.repo.IsExternalIDKnown(ctx, row.ExternalID)
|
||||
}
|
||||
return s.repo.IsDuplicate(ctx, row.Date, row.Description, row.Amount)
|
||||
}
|
||||
Reference in New Issue
Block a user