From 2b9029464d5da920b5551d5d0a997b53ba513e3b Mon Sep 17 00:00:00 2001 From: HeshamTB Date: Wed, 11 Mar 2026 18:51:15 +0300 Subject: [PATCH] init: - [WIP] database migrations Signed-off-by: HeshamTB --- .gitignore | 3 + README | 31 ++++++ README.md | 1 + cmd/sft/sft.go | 225 +++++++++++++++++++++++++++++++++++++++ go.mod | 5 + migrations/0_init.sql | 11 ++ migrations/1_rev1.sql | 10 ++ migrations/2_rev2.sql | 10 ++ migrations/3_rev3.sql | 9 ++ migrations/4_rev4.sql | 8 ++ migrations/migrations.go | 7 ++ 11 files changed, 320 insertions(+) create mode 100644 .gitignore create mode 100644 README create mode 120000 README.md create mode 100644 cmd/sft/sft.go create mode 100644 go.mod create mode 100644 migrations/0_init.sql create mode 100644 migrations/1_rev1.sql create mode 100644 migrations/2_rev2.sql create mode 100644 migrations/3_rev3.sql create mode 100644 migrations/4_rev4.sql create mode 100644 migrations/migrations.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9a1eaac --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ + +go.sum +cmd/sft/data.db diff --git a/README b/README new file mode 100644 index 0000000..2b84954 --- /dev/null +++ b/README @@ -0,0 +1,31 @@ + +2026-03-05 + +Secure File Transfer + +A secure file transfer in/out critical infra. SFT acts as the sole gateway for +file transfer. + +High level + Given a protected IT/OT env, an SFT node allows a login using LDAP or other + auth method to whitelist or all users in an auth domain, and transfer files + + No clustering for now. + Componenets: + - Main server handles + - Auth Confing + - Media trust + - File quarintne and scannning + - SFT Node trust + - User interfaces/API + + - SFT Node + - Auth users + - File transfer + + + SFT Servers seperate a low protection env to a critical env. Transfer files + into a temporerey env for scanning. + + Possible to integrate third-party scanning tools. Must be able to run + without internet, or at least scan locally while connected to internet. diff --git a/README.md b/README.md new file mode 120000 index 0000000..100b938 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/cmd/sft/sft.go b/cmd/sft/sft.go new file mode 100644 index 0000000..d14dfee --- /dev/null +++ b/cmd/sft/sft.go @@ -0,0 +1,225 @@ +package main + +import ( + "database/sql" + "io" + "log/slog" + "os" + "strconv" + "strings" + + "gitea.hbanafa.com/HeshamTB/sft/migrations" + _ "github.com/mattn/go-sqlite3" +) + + +var rootLog slog.Logger +const EXIT_FAIL = 1 +const FILE_DB_REV0 = "0_init.sql" +const DB_PATH = "./data.db" +const DB_DRIVER = "sqlite3" + +/* +Entry for SFT server + - WebUI + - File Quarintine and scanning + - DB + - LDAP Auth +*/ +func main() { + + lopt := slog.HandlerOptions{ Level: slog.LevelDebug } + lhan := slog.NewTextHandler(os.Stdout, &lopt) + rootLog := slog.New(lhan) + + rootLog.Info("init sft server") + + + // TODO: Init DB + + // This creates the db file if it does not exist + db, err := sql.Open(DB_DRIVER, DB_PATH) + if err != nil { + rootLog.Error("could not open connection to db", "err", err.Error()) + os.Exit(EXIT_FAIL) + } + defer db.Close() + db.SetMaxOpenConns(0) + + +// sqlStmt := ` +// create table if not exists _sft_db (db_rev integer not null, sft_ver text); +// delete from _sft_db; +// ` +// +// _, err = db.Exec(sqlStmt) +// if err != nil { +// rootLog.Error("error in init db", "err", err.Error()) +// os.Exit(EXIT_FAIL) +// } + + /* + - Case 1: Database file does not exist. Created with sql.Open() + - Case 2: Database file exists but table _sft_db does is not created. Created using 0_init.sql + 0_init.sql is always run + - Case 3: 0_init.sql is run, but next does not point to 1_*.sql. + creating a migration needs to run sql commands. + Or, check all migrations if they exist in the table, do not run. + It is critical to run in order. In this case we don't need the next column + */ + + // TODO: apply rev 0 + f, _ := migrations.DbMigrations.Open("0_init.sql") + defer f.Close() + + initSql, err := io.ReadAll(f) + if err != nil { + rootLog.Error("can not read embedFS db rev0", "err", err.Error()) + os.Exit(EXIT_FAIL) + } + + rootLog.Debug("migrations read", "bytes", len(initSql)) + + _, err = db.Exec(string(initSql)) + if err != nil { + rootLog.Error("failed to exec db init for rev0", "err", err.Error()) + rootLog.Warn("assume rev0 applied") + } + + rootLog.Info("db initilized") + + + rootLog.Debug("read on _sft_db") + + rows, err := db.Query("select db_rev, sft_ver from _sft_db;") + if err != nil { + rootLog.Error("error in init db", "err", err.Error()) + os.Exit(EXIT_FAIL) + } + + if !rows.Next() { + rootLog.Warn("db not migrated to 1") + } + rows.Close() + + // At this point we don't have fixed file names for migrations. Use prefix to increment + // Read all file names in embedFS and Query all migrations in _sft_db + + rows, err = db.Query("SELECT db_rev FROM _sft_db;") + if err != nil { + rootLog.Error("error", "err", err.Error()) + os.Exit(EXIT_FAIL) + } + + var appliedMigs []int + + for rows.Next() { + var val int + + if err := rows.Scan(&val); err != nil { + rootLog.Error("revision stored in db is not an integer") + os.Exit(EXIT_FAIL) + } + + rootLog.Debug("found applied database revision", "rev", val) + + appliedMigs = append(appliedMigs, val) + } + rows.Close() + + + migrationFilesDir, err := migrations.DbMigrations.ReadDir(".") + if err != nil { + rootLog.Error("could not list files in migrations embedFS", "err", err.Error()) + os.Exit(EXIT_FAIL) + } + + type Migration struct { + Filename string + Rev int + Data []byte + } + + var migs []Migration + + for i, file := range migrationFilesDir { + if file.IsDir() { + rootLog.Debug("migration dir. skipping", "name", file.Name()) + continue + } + rootLog.Debug("migration file", "name", file.Name()) + migStmts, err := migrations.DbMigrations.ReadFile(file.Name()) + if err != nil { + rootLog.Error("could not read migration file", "err", err.Error(), "name", file.Name()) + os.Exit(EXIT_FAIL) + } + + mRev, err := strconv.Atoi(strings.Split(file.Name(), "_")[0]) + if err != nil { + rootLog.Error( + "unexpected db revision. invalid file naming in migrations", + "err", err.Error(), + "name", file.Name()) + os.Exit(EXIT_FAIL) + } + + if mRev != i { + rootLog.Error("unexpected migration sequence", "found", mRev, "expected", i) + os.Exit(EXIT_FAIL) + } + + rev := Migration{ + Filename: file.Name(), + Data: migStmts, + Rev: mRev, + } + migs = append(migs, rev) + rootLog.Debug("loaded db migration", "name", rev.Filename) + + + // FIXME: this logic is bad when we have 2 or more migrations pending + dbTx, err := db.Begin() + if err != nil { + rootLog.Error("failed to start migration Tx") + os.Exit(EXIT_FAIL) + } + if rev.Rev >= len(appliedMigs) { + rootLog.Info("applying migration", "name", rev.Filename) + _, err = dbTx.Exec(string(rev.Data)) + if err != nil { + rootLog.Error("could not apply database migration", "err", err.Error()) + dbTx.Rollback() + os.Exit(EXIT_FAIL) + } + rootLog.Debug("committing migration", "name", rev.Filename) + rootLog.Info("applied migration", "name", rev.Filename) + } + + err = dbTx.Commit() + if err != nil { + rootLog.Error("failed to commit migration", "err", err.Error(), "name", rev.Filename) + os.Exit(EXIT_FAIL) + } + + + // NOTE: Expect migs to be ordered 0..1..2... + + } + rootLog.Info("database up-to-date") + + + // TODO: Create local admin users + // TODO: File transfer, quarintine, scan with local accounts + + + +} + +/* +initilize database with sft parameters and table from starting point +*/ +func initDb(db sql.DB) { + + +} + diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5084ff5 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module gitea.hbanafa.com/HeshamTB/sft + +go 1.25.6 + +require github.com/mattn/go-sqlite3 v1.14.34 // indirect diff --git a/migrations/0_init.sql b/migrations/0_init.sql new file mode 100644 index 0000000..0bd66ca --- /dev/null +++ b/migrations/0_init.sql @@ -0,0 +1,11 @@ + +/* SFT database migrations init */ + +CREATE TABLE IF NOT EXISTS _sft_db ( + db_rev INTEGER PRIMARY KEY, -- migration filename without sql suffix (.sql) + sft_ver TEXT, + applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +-- DELETE FROM _sft_db; + +INSERT INTO _sft_db (db_rev, sft_ver, applied_at) VALUES (0, "0.0.1-pre-alpha1", CURRENT_TIMESTAMP); diff --git a/migrations/1_rev1.sql b/migrations/1_rev1.sql new file mode 100644 index 0000000..03e2a9d --- /dev/null +++ b/migrations/1_rev1.sql @@ -0,0 +1,10 @@ + + +CREATE TABLE IF NOT EXISTS _sft_migtest_table ( + id integer PRIMARY KEY +); + +INSERT INTO _sft_migtest_table (id) VALUES (99); + +INSERT INTO _sft_db (db_rev, sft_ver, applied_at) VALUES (1, "0.0.1-pre-alpha1", CURRENT_TIMESTAMP); + diff --git a/migrations/2_rev2.sql b/migrations/2_rev2.sql new file mode 100644 index 0000000..d2137e9 --- /dev/null +++ b/migrations/2_rev2.sql @@ -0,0 +1,10 @@ + + + +CREATE TABLE IF NOT EXISTS _sft_migtest_table2 ( + id integer PRIMARY KEY +); + +INSERT INTO _sft_migtest_table (id) VALUES (42); + +INSERT INTO _sft_db (db_rev, sft_ver, applied_at) VALUES (2, "0.0.1-pre-alpha1", CURRENT_TIMESTAMP); diff --git a/migrations/3_rev3.sql b/migrations/3_rev3.sql new file mode 100644 index 0000000..7feaffb --- /dev/null +++ b/migrations/3_rev3.sql @@ -0,0 +1,9 @@ + + +CREATE TABLE IF NOT EXISTS _sft_migtest_table3 ( + id integer PRIMARY KEY +); + +INSERT INTO _sft_migtest_table (id) VALUES (42); + +INSERT INTO _sft_db (db_rev, sft_ver, applied_at) VALUES (3, "0.0.1-pre-alpha1", CURRENT_TIMESTAMP); diff --git a/migrations/4_rev4.sql b/migrations/4_rev4.sql new file mode 100644 index 0000000..7f441f8 --- /dev/null +++ b/migrations/4_rev4.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS _sft_migtest_table4 ( + id integer PRIMARY KEY +); + +INSERT INTO _sft_migtest_table (id) VALUES (42); + +INSERT INTO _sft_db (db_rev, sft_ver, applied_at) VALUES (4, "0.0.1-pre-alpha1", CURRENT_TIMESTAMP); + diff --git a/migrations/migrations.go b/migrations/migrations.go new file mode 100644 index 0000000..e8161fe --- /dev/null +++ b/migrations/migrations.go @@ -0,0 +1,7 @@ +package migrations + +import "embed" + + +//go:embed *.sql +var DbMigrations embed.FS