diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..6ede911 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,13 @@ +build: + image: golang + commands: + - make test + +notify: + slack: + webhook_url: $$SLACK_WEBHOOK + channel: general + username: $$SLACK_USERNAME + template: > + [{{ repo.full_name }}] <{{ system.link_url }}/{{ repo.full_name }}/{{ build.number }}|build #{{ build.number }}> on branch *{{ build.branch }}* by *{{ build.author }}* finished with a *{{ build.status }}* status. + diff --git a/Makefile b/Makefile index 9698311..3813e31 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ build: clean deps clean: rm -rf $(DIST_DIR) -test: +test: clean deps @go test -run=. -test.v ./config @go test -run=. -test.v ./database @go test -run=. -test.v ./dest diff --git a/README.md b/README.md index 9c1754c..2e47307 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ $ curl -s https://packagecloud.io/install/repositories/gregarmer/packages/script ## Getting Started 1. Follow the installation steps above. -2. Configure ~/.s3pgbackups - something like this: +2. Configure ~/.s3pgbackups or your-config.cfg - something like this: ```json { @@ -41,27 +41,31 @@ $ curl -s https://packagecloud.io/install/repositories/gregarmer/packages/script "pg_username": "username", "pg_password": "password", "pg_sslmode": true, - "pg_exclude_dbs": ["postgres", "template0"], - "pg_exclude_tables": ["django_session"] + "excludes": ["postgres", "template0", "*.django_session", "db1.table3"], } ``` > Note: Running s3pgbackups without a config will create a blank config. -3. Run `s3pgbackups` +3. Run `s3pgbackups -c your-config.cfg` or alternatively `s3pgbackups` ## Usage ``` $ s3pgbackups -h -Usage of s3pgbackups: - -n=false: don't actually do anything, just print what would be done - -v=false: be verbose +Usage of dist/s3pgbackups: + -c string path to the config file (default "~/.s3pgbackups") + -n don't actually do anything, just print what would be done + -v be verbose ``` -You can use the `-n` parameter to put s3pgbackups into no-op mode, where no actions -will actually be performed. +`-c config.cfg` is optional. If you don't specify a config it'll default to +`~/.s3pgbackups` and will create this file if it doesn't already exist. -The `-v` parameter will make s3pgbackups be verbose about what is actually happening. +You can use the `-n` parameter to put s3pgbackups into no-op mode, where no +actions will actually be performed. + +The `-v` parameter will make s3pgbackups be verbose about what is actually +happening. ## TODO diff --git a/config/config.go b/config/config.go index 1d1c954..a961f1e 100644 --- a/config/config.go +++ b/config/config.go @@ -3,6 +3,7 @@ package config import ( "encoding/json" "errors" + "fmt" "github.com/gregarmer/s3pgbackups/utils" "io/ioutil" "os" @@ -22,20 +23,13 @@ type Config struct { S3RotateOld bool `json:"s3_rotate_old"` // PostgreSQL - PostgresUsername string `json:"pg_username"` - PostgresPassword string `json:"pg_password"` - PostgresSSL bool `json:"pg_sslmode"` - PostgresExcludeDb []string `json:"pg_exclude_dbs"` - PostgresExcludeTable []string `json:"pg_exclude_tables"` -} + PostgresUsername string `json:"pg_username"` + PostgresPassword string `json:"pg_password"` + PostgresSSL bool `json:"pg_sslmode"` -func _shouldExclude(item string, excludes []string) bool { - for _, b := range excludes { - if b == item { - return true - } - } - return false + // New style excludes - see issue #1 - example: + // ["database.*", "*.table", "database"] + Excludes []string `json:"excludes"` } func (c *Config) Copy() Config { @@ -43,11 +37,27 @@ func (c *Config) Copy() Config { } func (c *Config) ShouldExcludeDb(db string) bool { - return _shouldExclude(db, c.PostgresExcludeDb) + for _, cmp := range c.Excludes { + if cmp == fmt.Sprintf("%s.*", db) || cmp == db { + return true + } + } + return false } -func (c *Config) ShouldExcludeTable(table string) bool { - return _shouldExclude(table, c.PostgresExcludeTable) +func (c *Config) ShouldExcludeTable(db string, t string) bool { + for _, cmp := range c.Excludes { + // all tables are excluded for this db + if cmp == fmt.Sprintf("%s.*", db) { + return true + } + + // table match is excluded for this db + if cmp == fmt.Sprintf("%s.%s", db, t) || cmp == fmt.Sprintf("*.%s", t) { + return true + } + } + return false } func (c *Config) PreFlight() error { @@ -68,16 +78,23 @@ func GetConfigPath() string { return filepath.Join(u.HomeDir, configFile) } -func LoadConfig() *Config { +func LoadConfig(configFile string) *Config { + // make sure the config file actually exists + if _, err := os.Stat(configFile); os.IsNotExist(err) { + utils.Fatalf("couldn't load config from %s", configFile) + } + // init config if needed - configPath := GetConfigPath() - if _, err := os.Stat(configPath); os.IsNotExist(err) { - InitConfig() - utils.Fatalf("couldn't find config, created empty config at %s, please configure", configPath) + defaultConfigPath := GetConfigPath() + if configFile == defaultConfigPath { + if _, err := os.Stat(configFile); os.IsNotExist(err) { + InitConfig() + utils.Fatalf("created empty config at %s, please configure", configFile) + } } // load config - file, _ := os.Open(configPath) + file, _ := os.Open(configFile) decoder := json.NewDecoder(file) config := Config{} if err := decoder.Decode(&config); err != nil { diff --git a/config/config_test.go b/config/config_test.go index 1581c77..a181981 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -18,3 +18,74 @@ func TestCopy(t *testing.T) { t.Fatalf("config.Copy() should return a copy that doesn't affect the original") } } + +func TestShouldExcludeDbWithNoExcludes(t *testing.T) { + conf := Config{} + var testDatabase string + + // includes + testDatabase = "foobar" + if conf.ShouldExcludeDb(testDatabase) != false { + t.Fatalf("%s should not have been excluded.", testDatabase) + } +} + +func TestShouldExcludeDb(t *testing.T) { + conf := Config{Excludes: []string{"foo", "*.bar", "baz.*", "quux.*"}} + var testDatabase string + + // excludes + testDatabase = "foo" + if conf.ShouldExcludeDb(testDatabase) != true { + t.Fatalf("%s should have been excluded.", testDatabase) + } + + testDatabase = "quux" + if conf.ShouldExcludeDb(testDatabase) != true { + t.Fatalf("%s should have been excluded.", testDatabase) + } + + // includes + testDatabase = "foobar" + if conf.ShouldExcludeDb(testDatabase) != false { + t.Fatalf("%s should not have been excluded.", testDatabase) + } +} + +func TestShouldExcludeTable(t *testing.T) { + conf := Config{Excludes: []string{"foo", "*.bar", "baz.*", "quux.gah"}} + var testDatabase string + var testTable string + + // excludes + testDatabase = "foo" + testTable = "bar" + if conf.ShouldExcludeTable(testDatabase, testTable) != true { + t.Fatalf("%s.%s should have been excluded.", testDatabase, testTable) + } + + testDatabase = "baz" + testTable = "quux" + if conf.ShouldExcludeTable(testDatabase, testTable) != true { + t.Fatalf("%s.%s should have been excluded.", testDatabase, testTable) + } + + testDatabase = "quux" + testTable = "gah" + if conf.ShouldExcludeTable(testDatabase, testTable) != true { + t.Fatalf("%s.%s should have been excluded.", testDatabase, testTable) + } + + // includes + testDatabase = "foo" + testTable = "gah" + if conf.ShouldExcludeTable(testDatabase, testTable) != false { + t.Fatalf("%s.%s should not have been excluded.", testDatabase, testTable) + } + + testDatabase = "foobar" + testTable = "gah" + if conf.ShouldExcludeTable(testDatabase, testTable) != false { + t.Fatalf("%s.%s should not have been excluded.", testDatabase, testTable) + } +} diff --git a/database/postgres.go b/database/postgres.go index e9727f1..9ca95d5 100644 --- a/database/postgres.go +++ b/database/postgres.go @@ -17,8 +17,7 @@ type Postgres struct { Config *config.Config } -func (postgres *Postgres) GetDatabases() []string { - var databases []string +func (postgres *Postgres) GetDSN() string { var sslmode string if postgres.Config.PostgresSSL { @@ -32,7 +31,13 @@ func (postgres *Postgres) GetDatabases() []string { postgres.Config.PostgresPassword, sslmode) - db, err := sql.Open("postgres", dsn) + return dsn +} + +func (postgres *Postgres) GetDatabases() []string { + var databases []string + + db, err := sql.Open("postgres", postgres.GetDSN()) utils.CheckErr(err) rows, err := db.Query("SELECT datname FROM pg_database") @@ -52,13 +57,48 @@ func (postgres *Postgres) GetDatabases() []string { return databases } +func (postgres *Postgres) GetTables(database string) []string { + var tables []string + + db, err := sql.Open(database, postgres.GetDSN()) + utils.CheckErr(err) + + tablesQuery := "SELECT table_name FROM information_schema.tables " + tablesQuery += "WHERE table_schema = 'public'" + rows, err := db.Query(tablesQuery) + utils.CheckErr(err) + defer rows.Close() + + for rows.Next() { + var name string + err = rows.Scan(&name) + utils.CheckErr(err) + tables = append(tables, name) + } + + err = rows.Err() // get any error encountered during iteration + utils.CheckErr(err) + + return tables +} + func (postgres *Postgres) DumpDatabase(db, workingDir string) string { backupFileName := fmt.Sprintf("%s-%s.sql", db, time.Now().Format("2006-01-02")) - pgDumpCmd := fmt.Sprintf("-E UTF-8 -T %s -f %s %s", - strings.Join(postgres.Config.PostgresExcludeTable, " -T "), - fmt.Sprintf("%s/%s", workingDir, backupFileName), - db) + + pgDumpCmd := fmt.Sprintf("-E UTF-8 -f %s ", + fmt.Sprintf("%s/%s", workingDir, backupFileName)) + + if postgres.Config.Excludes != nil { + for _, table := range postgres.GetTables(db) { + if postgres.Config.ShouldExcludeTable(db, table) { + pgDumpCmd += fmt.Sprintf("-T %s ", table) + } + } + } + + pgDumpCmd += db + log.Printf("executing pg_dump %s", pgDumpCmd) cmd := exec.Command("pg_dump", strings.Split(pgDumpCmd, " ")...) var out bytes.Buffer diff --git a/database/postgres_test.go b/database/postgres_test.go index de303fc..9bd6e42 100644 --- a/database/postgres_test.go +++ b/database/postgres_test.go @@ -1,6 +1,35 @@ package database -import "testing" +import ( + "github.com/gregarmer/s3pgbackups/config" + "testing" +) -func TestBasic(t *testing.T) { +var testConfig = config.Config{ + AwsAccessKey: "access", + AwsSecretKey: "secret", + PostgresUsername: "foo", + PostgresPassword: "bar", +} + +func TestGetDSNWithSSL(t *testing.T) { + testConfig.PostgresSSL = true + postgres := Postgres{Config: &testConfig} + expectedDsn := "user=foo password=bar dbname=postgres sslmode=require" + actualDsn := postgres.GetDSN() + if expectedDsn != actualDsn { + t.Fatalf("GetDSN with SSL failed. Expected '%s' got '%s'", + expectedDsn, actualDsn) + } +} + +func TestGetDSNWithoutSSL(t *testing.T) { + testConfig.PostgresSSL = false + postgres := Postgres{Config: &testConfig} + expectedDsn := "user=foo password=bar dbname=postgres sslmode=disable" + actualDsn := postgres.GetDSN() + if expectedDsn != actualDsn { + t.Fatalf("GetDSN without SSL failed. Expected '%s' got '%s'", + expectedDsn, actualDsn) + } } diff --git a/main.go b/main.go index 0017efe..96bcfe0 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ import ( const workingDir = "temp" +var configFile = flag.String("c", "~/.s3pgbackups", "path to the config file") var verbose = flag.Bool("v", false, "be verbose") var noop = flag.Bool("n", false, "don't actually do anything, just print what would be done") @@ -36,7 +37,7 @@ func main() { log.Printf("running in no-op mode, no commands will really be executed") } - conf := config.LoadConfig() + conf := config.LoadConfig(*configFile) // Don't print real passwords and secret keys in verbose mode verbose_config := conf.Copy() diff --git a/utils/errors_test.go b/utils/errors_test.go index 7ddc9fc..45bcd8f 100644 --- a/utils/errors_test.go +++ b/utils/errors_test.go @@ -1,6 +1,27 @@ package utils -import "testing" +import ( + "errors" + "os" + "os/exec" + "testing" +) -func TestBasic(t *testing.T) { +func TestCheckErr(t *testing.T) { + if os.Getenv("BE_CRASHER") == "1" { + fakeError := errors.New("Fake Error") + CheckErr(fakeError) + return + } + cmd := exec.Command(os.Args[0], "-test.run=TestCheckErr") + cmd.Env = append(os.Environ(), "BE_CRASHER=1") + err := cmd.Run() + if e, ok := err.(*exec.ExitError); ok && !e.Success() { + return + } + t.Fatalf("process ran with err %v, want exit status 1", err) +} + +func TestCheckErrWithoutError(t *testing.T) { + CheckErr(nil) }