vault backup: 2024-12-15 15:57:01

.obsidian/workspace.json

@@ -13,12 +13,12 @@
             "state": {
               "type": "markdown",
               "state": {
-                "file": "2024/Postgres and ZFS snapshots.md",
+                "file": "2024/Postgres Backups and ZFS snapshots.md",
                 "mode": "source",
                 "source": false
               },
               "icon": "lucide-file",
-              "title": "Postgres and ZFS snapshots"
+              "title": "Postgres Backups and ZFS snapshots"
             }
           },
           {
@@ -119,7 +119,7 @@
             "state": {
               "type": "backlink",
               "state": {
-                "file": "2024/Postgres and ZFS snapshots.md",
+                "file": "2024/Postgres Backups and ZFS snapshots.md",
                 "collapseAll": false,
                 "extraContext": false,
                 "sortOrder": "alphabetical",
@@ -129,7 +129,7 @@
                 "unlinkedCollapsed": true
               },
               "icon": "links-coming-in",
-              "title": "Backlinks for Postgres and ZFS snapshots"
+              "title": "Backlinks for Postgres Backups and ZFS snapshots"
             }
           },
           {
@@ -138,12 +138,12 @@
             "state": {
               "type": "outgoing-link",
               "state": {
-                "file": "2024/Postgres and ZFS snapshots.md",
+                "file": "2024/Postgres Backups and ZFS snapshots.md",
                 "linksCollapsed": false,
                 "unlinkedCollapsed": true
               },
               "icon": "links-going-out",
-              "title": "Outgoing links from Postgres and ZFS snapshots"
+              "title": "Outgoing links from Postgres Backups and ZFS snapshots"
             }
           },
           {
@@ -165,10 +165,10 @@
             "state": {
               "type": "outline",
               "state": {
-                "file": "2024/Postgres and ZFS snapshots.md"
+                "file": "2024/Postgres Backups and ZFS snapshots.md"
               },
               "icon": "lucide-list",
-              "title": "Outline of Postgres and ZFS snapshots"
+              "title": "Outline of Postgres Backups and ZFS snapshots"
             }
           }
         ]
@@ -190,8 +190,12 @@
   },
   "active": "abe7f08be8c489db",
   "lastOpenFiles": [
+    "2024/Postgres and ZFS snapshots/postgres.go",
+    "2024/Postgres and ZFS snapshots/postgres (1).go",
+    "2024/Postgres and ZFS snapshots",
+    "2024/untitled folder",
     "2024/So you wanna do FreeBSD 14.1 and native Home Assistant?.md",
-    "2024/Postgres and ZFS snapshots.md",
+    "2024/Postgres Backups and ZFS snapshots.md",
     "2024/teledyne-cybersecurity-mail-ssh-traffic-20240815-unenc.pdf",
     "2024/verjaardag-uitnodiging-jerry-35j-23-mrt-2024.png",
     "2024/verjaardags-feest-35-jaar-in-breugel.md",
@@ -209,7 +213,6 @@
     "2024/freebsd-jail-vanilla.md",
     "2024/happy-holidays-nye-2024-kaart/IMG_E7695.HEIC",
     "2024/happy-holidays-nye-2024-kaart",
-    "2024/untitled folder",
     "2024/mijnpositievegezondheid.net/Mijn Positieve Gezondheid - 2024-11-08.pdf",
     "2024/mijnpositievegezondheid.net",
     "2024/photoprism.md",
@@ -217,7 +220,6 @@
     "2019/Nieuwsbrief #24 - knoflook kweken.pdf",
     "2024/git-svn mirror.md",
     "2024/Batterij apparaten voor opladen.md",
-    "2024/Untitled",
     "2024-08-03.md",
     "2021/vakantie-liefde-op-terschelling-zomer-2019.md",
     "2024/mac-reinstall-notes.md",
@@ -228,14 +230,12 @@
     "2024/verjaardagskaart-pap-59-met-jerry-erbij/DSC07975.JPG",
     "2024/verjaardagskaart-pap-59-met-jerry-erbij/DSC03175.JPG",
     "2024/verjaardagskaart-pap-59-met-jerry-erbij/20240331_102549_jerry_leon.jpg",
-    "2024/verjaardagskaart-pap-59-met-jerry-erbij",
     "2024/verjaardagskaart-pap-met-jerry/DSC03175.JPG",
     "2024/Gezonde routines boek review & notes.md",
     "2024/Brouwsels 2024.md",
     "2021/Tagliatelle-Bolognese.md",
     "2024/gyros-kruiden-jacobus.md",
     "2024/POP gesprek 2024-Q1.md",
-    "2024/Neways introductie interview voorbereiding.md",
     "Untitled.canvas"
   ]
 }

2024/Postgres Backups and ZFS snapshots.md

@@ -1,5 +1,9 @@
+
 * https://github.com/jibudata/amberapp/blob/34852920052c58f3d518a28e4d09a50584cf40f8/pkg/postgres/postgres.go
 * Quiesce & Unquiesce
+* https://zrepl.github.io/v0.2.1/configuration/snapshotting.html#postgres-checkpoint-hook
+* https://www.postgresql.org/docs/16/wal-reliability.html
+* https://www.postgresql.org/docs/15/functions-admin.html
 
 ```go
 func takeSnapshot(db *sql.DB, dataset string, height int, tag string) error {

2024/Postgres and ZFS snapshots/postgres.go

@@ -0,0 +1,222 @@
+package postgres
+
+import (
+	"database/sql"
+	"fmt"
+	"strconv"
+	"strings"
+
+	_ "github.com/lib/pq"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	"github.com/jibudata/amberapp/api/v1alpha1"
+	"github.com/jibudata/amberapp/pkg/appconfig"
+)
+
+const (
+	// < v15.0
+	PG_START_BACKUP = "pg_start_backup"
+	PG_STOP_BACKUP  = "pg_stop_backup"
+	// >= v15.0
+	PG_BACKUP_START = "pg_backup_start"
+	PG_BACKUP_STOP  = "pg_backup_stop"
+)
+
+type PG struct {
+	config  appconfig.Config
+	db      *sql.DB
+	version string
+}
+
+func (pg *PG) Init(appConfig appconfig.Config) error {
+	pg.config = appConfig
+	return nil
+}
+
+func (pg *PG) Connect() error {
+	var err error
+	log.Log.Info("postgres connecting")
+
+	connectionConfigStrings := pg.getConnectionString()
+	if len(connectionConfigStrings) == 0 {
+		return fmt.Errorf("no database found in %s", pg.config.Name)
+	}
+
+	for i := 0; i < len(connectionConfigStrings); i++ {
+		pg.db, err = sql.Open("postgres", connectionConfigStrings[i])
+		if err != nil {
+			log.Log.Error(err, "cannot connect to postgres")
+			return err
+		}
+
+		err = pg.db.Ping()
+		if err != nil {
+			log.Log.Error(err, fmt.Sprintf("cannot connect to postgres database %s", pg.config.Databases[i]))
+			return err
+		}
+
+		err = pg.getVersion()
+		if err != nil {
+			return err
+		}
+
+		pg.db.Close()
+	}
+	log.Log.Info("connected to postgres")
+	return nil
+}
+
+func (pg *PG) Prepare() (*v1alpha1.PreservedConfig, error) {
+	return nil, nil
+}
+
+func (pg *PG) Quiesce() (*v1alpha1.QuiesceResult, error) {
+	var err error
+	log.Log.Info("postgres quiesce in progress...")
+
+	backupName := "test"
+	fastStartString := "true"
+
+	connectionConfigStrings := pg.getConnectionString()
+	if len(connectionConfigStrings) == 0 {
+		return nil, fmt.Errorf("no database found in %s", pg.config.Name)
+	}
+
+	for i := 0; i < len(connectionConfigStrings); i++ {
+		pg.db, err = sql.Open("postgres", connectionConfigStrings[i])
+		if err != nil {
+			log.Log.Error(err, "cannot connect to postgres")
+			return nil, err
+		}
+
+		queryStr := fmt.Sprintf("select %s('%s', %s);", pg.getQuiesceCmd(), backupName, fastStartString)
+
+		result, queryErr := pg.db.Query(queryStr)
+
+		if queryErr != nil {
+			if strings.Contains(queryErr.Error(), "backup is already in progress") {
+				pg.db.Close()
+				continue
+			}
+			log.Log.Error(queryErr, "could not start postgres backup")
+			return nil, queryErr
+		}
+
+		var snapshotLocation string
+		result.Next()
+
+		scanErr := result.Scan(&snapshotLocation)
+		if scanErr != nil {
+			log.Log.Error(scanErr, "Postgres backup apparently started but could not understand server response")
+			return nil, scanErr
+		}
+		log.Log.Info(fmt.Sprintf("Successfully reach consistent recovery state at %s", snapshotLocation))
+		pg.db.Close()
+	}
+	return nil, nil
+}
+
+func (pg *PG) Unquiesce(prev *v1alpha1.PreservedConfig) error {
+	var err error
+	log.Log.Info("postgres unquiesce in progress...")
+	connectionConfigStrings := pg.getConnectionString()
+	if len(connectionConfigStrings) == 0 {
+		return fmt.Errorf("no database found in %s", pg.config.Name)
+	}
+
+	for i := 0; i < len(connectionConfigStrings); i++ {
+		pg.db, err = sql.Open("postgres", connectionConfigStrings[i])
+		if err != nil {
+			log.Log.Error(err, "cannot connect to postgres")
+			return err
+		}
+		defer pg.db.Close()
+
+		result, queryErr := pg.db.Query(fmt.Sprintf("select %s();", pg.getUnQuiesceCmd()))
+		if queryErr != nil {
+			if strings.Contains(queryErr.Error(), "not in progress") {
+				pg.db.Close()
+				continue
+			}
+			log.Log.Error(queryErr, "could not stop backup")
+			return queryErr
+		}
+
+		var snapshotLocation string
+		result.Next()
+
+		scanErr := result.Scan(&snapshotLocation)
+		if scanErr != nil {
+			log.Log.Error(scanErr, "Postgres backup apparently stopped but could not understand server response")
+			return scanErr
+		}
+	}
+	return nil
+}
+
+func (pg *PG) getConnectionString() []string {
+	var dbname string
+	var connstr []string
+
+	if len(pg.config.Databases) == 0 {
+		log.Log.Error(fmt.Errorf("no database found in %s", pg.config.Name), "")
+		return connstr
+	}
+
+	for i := 0; i < len(pg.config.Databases); i++ {
+		dbname = pg.config.Databases[i]
+		connstr = append(connstr, fmt.Sprintf("host=%s user=%s password=%s dbname=%s sslmode=disable", pg.config.Host, pg.config.Username, pg.config.Password, dbname))
+	}
+	return connstr
+}
+
+func (pg *PG) getVersion() error {
+	var version string
+	result, queryErr := pg.db.Query("show server_version;")
+	if queryErr != nil {
+		log.Log.Error(queryErr, "could get postgres version")
+		return queryErr
+	}
+
+	result.Next()
+	scanErr := result.Scan(&version)
+	if scanErr != nil {
+		log.Log.Error(scanErr, "scan postgres version with error")
+		return scanErr
+	}
+
+	pg.version = strings.Split(version, " ")[0]
+	log.Log.Info("get postgres version", "version", version, "instance", pg.config.Name)
+
+	return nil
+}
+
+func (pg *PG) isVersionAboveV15() bool {
+	aboveV15 := false
+	if pg.version != "" {
+		version, err := strconv.ParseFloat(pg.version, 64)
+		if err != nil {
+			log.Log.Error(err, "failed to convert version to number", "version", pg.version)
+		} else {
+			if version >= 15.0 {
+				aboveV15 = true
+			}
+		}
+	}
+
+	return aboveV15
+}
+
+func (pg *PG) getQuiesceCmd() string {
+	if pg.isVersionAboveV15() {
+		return PG_BACKUP_START
+	}
+	return PG_START_BACKUP
+}
+
+func (pg *PG) getUnQuiesceCmd() string {
+	if pg.isVersionAboveV15() {
+		return PG_BACKUP_STOP
+	}
+	return PG_STOP_BACKUP
+}