From 0e43d5ed69d6895b1da8eab795edd3c85b2875ff Mon Sep 17 00:00:00 2001 From: NiseVoid Date: Wed, 4 Jul 2018 13:41:59 +0200 Subject: [PATCH 1/5] Remove newer database check --- migrate.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/migrate.go b/migrate.go index 465a96b..dcc4b29 100644 --- a/migrate.go +++ b/migrate.go @@ -21,9 +21,6 @@ const DefaultTableName = `version` // ErrUpdatesMissing indicates an update is missing, making it impossible to execute the migration var ErrUpdatesMissing = errors.New(`Missing migration files`) -// ErrDatabaseNewer indicates that the database version is newer than the requested version. We throw an error because downgrades might cause dataloss -var ErrDatabaseNewer = errors.New(`Current version is newer than the requested version`) - const fileFormat = `%04d.sql` // AssetFunc is a function that returns the data for the given name @@ -69,10 +66,6 @@ func Migrate(db *sql.DB, version int, o Options, asset AssetFunc) error { return err } - if v > version { - return ErrDatabaseNewer - } - for i := v + 1; i <= version; i++ { script, err := asset(fmt.Sprintf(o.AssetPrefix+fileFormat, i)) if err != nil { From de9e7d9a41c73b7144e10e3a1a73af683e3114a5 Mon Sep 17 00:00:00 2001 From: NiseVoid Date: Mon, 28 Oct 2019 13:01:29 +0100 Subject: [PATCH 2/5] Add filename to error message --- migrate.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/migrate.go b/migrate.go index dcc4b29..96bd542 100644 --- a/migrate.go +++ b/migrate.go @@ -67,19 +67,22 @@ func Migrate(db *sql.DB, version int, o Options, asset AssetFunc) error { } for i := v + 1; i <= version; i++ { - script, err := asset(fmt.Sprintf(o.AssetPrefix+fileFormat, i)) + fileName := fmt.Sprintf(o.AssetPrefix+fileFormat, i) + errorf := func(e error) error { return fmt.Errorf(`migration "%s" failed: %w`, fileName, e) } + + script, err := asset(fileName) if err != nil { - return ErrUpdatesMissing + return errorf(ErrUpdatesMissing) } _, err = tx.Exec(string(script)) if err != nil { - return err + return errorf(err) } _, err = tx.Exec(`INSERT INTO ` + o.TableName + ` VALUES (` + strconv.Itoa(i) + `)`) if err != nil { - return err + return errorf(err) } } From f39c1c640a27e1c3aba70c3de29aa0180ca2b396 Mon Sep 17 00:00:00 2001 From: NiseVoid Date: Tue, 12 Nov 2019 13:01:36 +0100 Subject: [PATCH 3/5] Support non-PostgreSQL databases --- migrate.go | 38 ++++++++++++++++++++++---------------- query.go | 35 +++++++++++++++++++++++++++++++++++ util.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 16 deletions(-) create mode 100644 query.go create mode 100644 util.go diff --git a/migrate.go b/migrate.go index 96bd542..ab7984e 100644 --- a/migrate.go +++ b/migrate.go @@ -11,7 +11,7 @@ import ( // Options contains all settings type Options struct { TableName string // Name used for version info table; defaults to DefaultTableName if not set - Schema string // Schema used for version info table; For PostgreSQL, ignored if not set + Schema string // Schema used for version info table; In PostgreSQL the current schema is changed to the one specified here AssetPrefix string } @@ -29,23 +29,13 @@ type AssetFunc func(string) ([]byte, error) // Migrate executes migrations to get to the desired version // Downgrading is not supported as it could result in data loss func Migrate(db *sql.DB, version int, o Options, asset AssetFunc) error { + dbType := getDbType(db) if o.TableName == `` { o.TableName = DefaultTableName } var err error - searchPath := `public` - _ = db.QueryRow(`SHOW search_path`).Scan(&searchPath) - - if o.Schema != `` { - _, _ = db.Exec(`CREATE SCHEMA IF NOT EXISTS ` + o.Schema) - - _, err = db.Exec(`SET search_path TO ` + o.Schema) - if err != nil { - return err - } - } tx, err := db.Begin() if err != nil { @@ -53,12 +43,28 @@ func Migrate(db *sql.DB, version int, o Options, asset AssetFunc) error { } defer tx.Rollback() - _, err = tx.Exec(`CREATE TABLE IF NOT EXISTS ` + o.TableName + ` (Version integer NOT NULL PRIMARY KEY)`) + if o.Schema != `` { + err = createSchemaIfNotExists(tx, o.Schema) + if err != nil { + return err + } + + if dbType == `pq` { + _ = tx.QueryRow(`SHOW search_path`).Scan(&searchPath) + + _, err = tx.Exec(`SET search_path TO ` + o.Schema) + if err != nil { + return err + } + } + } + + versionTable, err := createTableIfNotExists(tx, o.Schema, o.TableName) if err != nil { return err } - row := tx.QueryRow(`SELECT Version FROM ` + o.TableName + ` ORDER BY Version DESC`) + row := tx.QueryRow(`SELECT version FROM ` + versionTable + ` ORDER BY Version DESC`) var v int err = row.Scan(&v) @@ -80,13 +86,13 @@ func Migrate(db *sql.DB, version int, o Options, asset AssetFunc) error { return errorf(err) } - _, err = tx.Exec(`INSERT INTO ` + o.TableName + ` VALUES (` + strconv.Itoa(i) + `)`) + _, err = tx.Exec(`INSERT INTO ` + versionTable + ` (version) VALUES (` + strconv.Itoa(i) + `)`) if err != nil { return errorf(err) } } - if o.Schema != `` { + if dbType == `pq` && o.Schema != `` { _, err = tx.Exec(`SET search_path TO ` + searchPath) if err != nil { return err diff --git a/query.go b/query.go new file mode 100644 index 0000000..b95419f --- /dev/null +++ b/query.go @@ -0,0 +1,35 @@ +package migrate + +import "database/sql" + +func createSchemaIfNotExists(tx *sql.Tx, schema string) error { + row := tx.QueryRow(`SELECT 1 FROM information_schema.schemata WHERE schema_name = '` + schema + `'`) + err := row.Scan(new(int)) + if err == sql.ErrNoRows { + _, err = tx.Exec(`CREATE SCHEMA ` + schema) + if err != nil { + return err + } + } + return err +} + +func createTableIfNotExists(tx *sql.Tx, schema, table string) (string, error) { + versionTable := table + var schemaCond string + + if schema != `` { + versionTable = schema + `.` + table + schemaCond = ` AND table_schema = '` + schema + `'` + } + + row := tx.QueryRow(`SELECT 1 FROM information_schema.tables WHERE table_name = '` + table + `'` + schemaCond) + err := row.Scan(new(int)) + if err == sql.ErrNoRows { + _, err = tx.Exec(`CREATE TABLE ` + versionTable + ` (version integer NOT NULL PRIMARY KEY)`) + if err != nil { + return ``, err + } + } + return versionTable, err +} diff --git a/util.go b/util.go new file mode 100644 index 0000000..c4bc8b0 --- /dev/null +++ b/util.go @@ -0,0 +1,29 @@ +package migrate + +import ( + "database/sql" + "reflect" + "strings" +) + +func getPkgName(db *sql.DB) string { + t := reflect.TypeOf(db.Driver()) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + path := t.PkgPath() + parts := strings.Split(path, `/vendor/`) + return parts[len(parts)-1] +} + +func getDbType(db *sql.DB) string { + switch getPkgName(db) { + case `github.com/lib/pq`, `github.com/jackc/pgx/stdlib`: + return `pq` + case `github.com/go-sql-driver/mysql`, `github.com/ziutek/mymysql/godrv`: + return `my` + case `github.com/denisenkom/go-mssqldb`: + return `ms` + } + return `` +} From 1bec960e38638e3ed14c93b9b5df6f4f307d1a0f Mon Sep 17 00:00:00 2001 From: NiseVoid Date: Wed, 24 Feb 2021 14:27:48 +0100 Subject: [PATCH 4/5] Use fs.FS --- go.mod | 3 +++ migrate.go | 21 +++++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 go.mod diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..909249c --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module git.fuyu.moe/Fuyu/migrate/v2 + +go 1.16 diff --git a/migrate.go b/migrate.go index 465a96b..9e902a8 100644 --- a/migrate.go +++ b/migrate.go @@ -5,14 +5,14 @@ import ( "database/sql" "errors" "fmt" + "io/fs" "strconv" ) // Options contains all settings type Options struct { - TableName string // Name used for version info table; defaults to DefaultTableName if not set - Schema string // Schema used for version info table; For PostgreSQL, ignored if not set - AssetPrefix string + TableName string // Name used for version info table; defaults to DefaultTableName if not set + Schema string // Schema used for version info table; For PostgreSQL, ignored if not set } // DefaultTableName is the name used when no TableName is specified in Options @@ -29,15 +29,20 @@ const fileFormat = `%04d.sql` // AssetFunc is a function that returns the data for the given name type AssetFunc func(string) ([]byte, error) -// Migrate executes migrations to get to the desired version +// Migrate executes all migrations +// Filenames need to have incrementing numbers // Downgrading is not supported as it could result in data loss -func Migrate(db *sql.DB, version int, o Options, asset AssetFunc) error { +func Migrate(db *sql.DB, o Options, assets fs.FS) error { + entries, err := fs.ReadDir(assets, `.`) + if err != nil { + panic(`failed to read list of files`) + } + version := len(entries) + if o.TableName == `` { o.TableName = DefaultTableName } - var err error - searchPath := `public` _ = db.QueryRow(`SHOW search_path`).Scan(&searchPath) @@ -74,7 +79,7 @@ func Migrate(db *sql.DB, version int, o Options, asset AssetFunc) error { } for i := v + 1; i <= version; i++ { - script, err := asset(fmt.Sprintf(o.AssetPrefix+fileFormat, i)) + script, err := fs.ReadFile(assets, fmt.Sprintf(fileFormat, i)) if err != nil { return ErrUpdatesMissing } From 9d4454628f9aa05ee77fbf0b89d6b9ae86f1545d Mon Sep 17 00:00:00 2001 From: NiseVoid Date: Wed, 24 Feb 2021 14:27:48 +0100 Subject: [PATCH 5/5] Use fs.FS --- go.mod | 3 +++ migrate.go | 31 ++++++++++++++++++------------- 2 files changed, 21 insertions(+), 13 deletions(-) create mode 100644 go.mod diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..909249c --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module git.fuyu.moe/Fuyu/migrate/v2 + +go 1.16 diff --git a/migrate.go b/migrate.go index ab7984e..b2c377a 100644 --- a/migrate.go +++ b/migrate.go @@ -5,14 +5,14 @@ import ( "database/sql" "errors" "fmt" + "io/fs" "strconv" ) // Options contains all settings type Options struct { - TableName string // Name used for version info table; defaults to DefaultTableName if not set - Schema string // Schema used for version info table; In PostgreSQL the current schema is changed to the one specified here - AssetPrefix string + TableName string // Name used for version info table; defaults to DefaultTableName if not set + Schema string // Schema used for version info table; In PostgreSQL the current schema is changed to the one specified here } // DefaultTableName is the name used when no TableName is specified in Options @@ -26,16 +26,15 @@ const fileFormat = `%04d.sql` // AssetFunc is a function that returns the data for the given name type AssetFunc func(string) ([]byte, error) -// Migrate executes migrations to get to the desired version +// Migrate executes all migrations +// Filenames need to have incrementing numbers // Downgrading is not supported as it could result in data loss -func Migrate(db *sql.DB, version int, o Options, asset AssetFunc) error { - dbType := getDbType(db) - if o.TableName == `` { - o.TableName = DefaultTableName +func Migrate(db *sql.DB, o Options, assets fs.FS) error { + entries, err := fs.ReadDir(assets, `.`) + if err != nil { + panic(`failed to read list of files`) } - - var err error - searchPath := `public` + version := len(entries) tx, err := db.Begin() if err != nil { @@ -43,6 +42,12 @@ func Migrate(db *sql.DB, version int, o Options, asset AssetFunc) error { } defer tx.Rollback() + dbType := getDbType(db) + searchPath := `public` + if o.TableName == `` { + o.TableName = DefaultTableName + } + if o.Schema != `` { err = createSchemaIfNotExists(tx, o.Schema) if err != nil { @@ -73,10 +78,10 @@ func Migrate(db *sql.DB, version int, o Options, asset AssetFunc) error { } for i := v + 1; i <= version; i++ { - fileName := fmt.Sprintf(o.AssetPrefix+fileFormat, i) + fileName := fmt.Sprintf(fileFormat, i) errorf := func(e error) error { return fmt.Errorf(`migration "%s" failed: %w`, fileName, e) } - script, err := asset(fileName) + script, err := fs.ReadFile(assets, fileName) if err != nil { return errorf(ErrUpdatesMissing) }