jerry
/
private-schrijfsels-en-notities
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

vault backup: 2024-12-15 15:57:01

This commit is contained in:
Jerry Jacobs 2024-12-15 15:57:01 +01:00
parent 5ed523e41f
commit ef9db5e65d
5 changed files with 2017 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
.obsidian/workspace.json vendored
View File

@ -13,12 +13,12 @@
"state": {
"type": "markdown",
"state": {
"file": "2024/Postgres and ZFS snapshots.md",
"file": "2024/Postgres Backups and ZFS snapshots.md",
"mode": "source",
"source": false
},
"icon": "lucide-file",
"title": "Postgres and ZFS snapshots"
"title": "Postgres Backups and ZFS snapshots"
}
},
{
@ -119,7 +119,7 @@
"state": {
"type": "backlink",
"state": {
"file": "2024/Postgres and ZFS snapshots.md",
"file": "2024/Postgres Backups and ZFS snapshots.md",
"collapseAll": false,
"extraContext": false,
"sortOrder": "alphabetical",
@ -129,7 +129,7 @@
"unlinkedCollapsed": true
},
"icon": "links-coming-in",
"title": "Backlinks for Postgres and ZFS snapshots"
"title": "Backlinks for Postgres Backups and ZFS snapshots"
}
},
{
@ -138,12 +138,12 @@
"state": {
"type": "outgoing-link",
"state": {
"file": "2024/Postgres and ZFS snapshots.md",
"file": "2024/Postgres Backups and ZFS snapshots.md",
"linksCollapsed": false,
"unlinkedCollapsed": true
},
"icon": "links-going-out",
"title": "Outgoing links from Postgres and ZFS snapshots"
"title": "Outgoing links from Postgres Backups and ZFS snapshots"
}
},
{
@ -165,10 +165,10 @@
"state": {
"type": "outline",
"state": {
"file": "2024/Postgres and ZFS snapshots.md"
"file": "2024/Postgres Backups and ZFS snapshots.md"
},
"icon": "lucide-list",
"title": "Outline of Postgres and ZFS snapshots"
"title": "Outline of Postgres Backups and ZFS snapshots"
}
}
]
@ -190,8 +190,12 @@
},
"active": "abe7f08be8c489db",
"lastOpenFiles": [
"2024/Postgres and ZFS snapshots/postgres.go",
"2024/Postgres and ZFS snapshots/postgres (1).go",
"2024/Postgres and ZFS snapshots",
"2024/untitled folder",
"2024/So you wanna do FreeBSD 14.1 and native Home Assistant?.md",
"2024/Postgres and ZFS snapshots.md",
"2024/Postgres Backups and ZFS snapshots.md",
"2024/teledyne-cybersecurity-mail-ssh-traffic-20240815-unenc.pdf",
"2024/verjaardag-uitnodiging-jerry-35j-23-mrt-2024.png",
"2024/verjaardags-feest-35-jaar-in-breugel.md",
@ -209,7 +213,6 @@
"2024/freebsd-jail-vanilla.md",
"2024/happy-holidays-nye-2024-kaart/IMG_E7695.HEIC",
"2024/happy-holidays-nye-2024-kaart",
"2024/untitled folder",
"2024/mijnpositievegezondheid.net/Mijn Positieve Gezondheid - 2024-11-08.pdf",
"2024/mijnpositievegezondheid.net",
"2024/photoprism.md",
@ -217,7 +220,6 @@
"2019/Nieuwsbrief #24 - knoflook kweken.pdf",
"2024/git-svn mirror.md",
"2024/Batterij apparaten voor opladen.md",
"2024/Untitled",
"2024-08-03.md",
"2021/vakantie-liefde-op-terschelling-zomer-2019.md",
"2024/mac-reinstall-notes.md",
@ -228,14 +230,12 @@
"2024/verjaardagskaart-pap-59-met-jerry-erbij/DSC07975.JPG",
"2024/verjaardagskaart-pap-59-met-jerry-erbij/DSC03175.JPG",
"2024/verjaardagskaart-pap-59-met-jerry-erbij/20240331_102549_jerry_leon.jpg",
"2024/verjaardagskaart-pap-59-met-jerry-erbij",
"2024/verjaardagskaart-pap-met-jerry/DSC03175.JPG",
"2024/Gezonde routines boek review & notes.md",
"2024/Brouwsels 2024.md",
"2021/Tagliatelle-Bolognese.md",
"2024/gyros-kruiden-jacobus.md",
"2024/POP gesprek 2024-Q1.md",
"2024/Neways introductie interview voorbereiding.md",
"Untitled.canvas"
]
}

BIN
2024/.DS_Store vendored
View File

Binary file not shown.

4
2024/Postgres and ZFS snapshots.md → 2024/Postgres Backups and ZFS snapshots.md
View File

@ -1,5 +1,9 @@
* https://github.com/jibudata/amberapp/blob/34852920052c58f3d518a28e4d09a50584cf40f8/pkg/postgres/postgres.go
* Quiesce & Unquiesce
* https://zrepl.github.io/v0.2.1/configuration/snapshotting.html#postgres-checkpoint-hook
* https://www.postgresql.org/docs/16/wal-reliability.html
* https://www.postgresql.org/docs/15/functions-admin.html
```go
func takeSnapshot(db *sql.DB, dataset string, height int, tag string) error {

1778
2024/Postgres and ZFS snapshots/postgres (1).go Normal file
View File

File diff suppressed because it is too large Load Diff

222
2024/Postgres and ZFS snapshots/postgres.go Normal file
View File

@ -0,0 +1,222 @@
package postgres
import (
"database/sql"
"fmt"
"strconv"
"strings"
_ "github.com/lib/pq"
"sigs.k8s.io/controller-runtime/pkg/log"
"github.com/jibudata/amberapp/api/v1alpha1"
"github.com/jibudata/amberapp/pkg/appconfig"
)
const (
// < v15.0
PG_START_BACKUP = "pg_start_backup"
PG_STOP_BACKUP = "pg_stop_backup"
// >= v15.0
PG_BACKUP_START = "pg_backup_start"
PG_BACKUP_STOP = "pg_backup_stop"
)
type PG struct {
config appconfig.Config
db *sql.DB
version string
}
func (pg *PG) Init(appConfig appconfig.Config) error {
pg.config = appConfig
return nil
}
func (pg *PG) Connect() error {
var err error
log.Log.Info("postgres connecting")
connectionConfigStrings := pg.getConnectionString()
if len(connectionConfigStrings) == 0 {
return fmt.Errorf("no database found in %s", pg.config.Name)
}
for i := 0; i < len(connectionConfigStrings); i++ {
pg.db, err = sql.Open("postgres", connectionConfigStrings[i])
if err != nil {
log.Log.Error(err, "cannot connect to postgres")
return err
}
err = pg.db.Ping()
if err != nil {
log.Log.Error(err, fmt.Sprintf("cannot connect to postgres database %s", pg.config.Databases[i]))
return err
}
err = pg.getVersion()
if err != nil {
return err
}
pg.db.Close()
}
log.Log.Info("connected to postgres")
return nil
}
func (pg *PG) Prepare() (*v1alpha1.PreservedConfig, error) {
return nil, nil
}
func (pg *PG) Quiesce() (*v1alpha1.QuiesceResult, error) {
var err error
log.Log.Info("postgres quiesce in progress...")
backupName := "test"
fastStartString := "true"
connectionConfigStrings := pg.getConnectionString()
if len(connectionConfigStrings) == 0 {
return nil, fmt.Errorf("no database found in %s", pg.config.Name)
}
for i := 0; i < len(connectionConfigStrings); i++ {
pg.db, err = sql.Open("postgres", connectionConfigStrings[i])
if err != nil {
log.Log.Error(err, "cannot connect to postgres")
return nil, err
}
queryStr := fmt.Sprintf("select %s('%s', %s);", pg.getQuiesceCmd(), backupName, fastStartString)
result, queryErr := pg.db.Query(queryStr)
if queryErr != nil {
if strings.Contains(queryErr.Error(), "backup is already in progress") {
pg.db.Close()
continue
}
log.Log.Error(queryErr, "could not start postgres backup")
return nil, queryErr
}
var snapshotLocation string
result.Next()
scanErr := result.Scan(&snapshotLocation)
if scanErr != nil {
log.Log.Error(scanErr, "Postgres backup apparently started but could not understand server response")
return nil, scanErr
}
log.Log.Info(fmt.Sprintf("Successfully reach consistent recovery state at %s", snapshotLocation))
pg.db.Close()
}
return nil, nil
}
func (pg *PG) Unquiesce(prev *v1alpha1.PreservedConfig) error {
var err error
log.Log.Info("postgres unquiesce in progress...")
connectionConfigStrings := pg.getConnectionString()
if len(connectionConfigStrings) == 0 {
return fmt.Errorf("no database found in %s", pg.config.Name)
}
for i := 0; i < len(connectionConfigStrings); i++ {
pg.db, err = sql.Open("postgres", connectionConfigStrings[i])
if err != nil {
log.Log.Error(err, "cannot connect to postgres")
return err
}
defer pg.db.Close()
result, queryErr := pg.db.Query(fmt.Sprintf("select %s();", pg.getUnQuiesceCmd()))
if queryErr != nil {
if strings.Contains(queryErr.Error(), "not in progress") {
pg.db.Close()
continue
}
log.Log.Error(queryErr, "could not stop backup")
return queryErr
}
var snapshotLocation string
result.Next()
scanErr := result.Scan(&snapshotLocation)
if scanErr != nil {
log.Log.Error(scanErr, "Postgres backup apparently stopped but could not understand server response")
return scanErr
}
}
return nil
}
func (pg *PG) getConnectionString() []string {
var dbname string
var connstr []string
if len(pg.config.Databases) == 0 {
log.Log.Error(fmt.Errorf("no database found in %s", pg.config.Name), "")
return connstr
}
for i := 0; i < len(pg.config.Databases); i++ {
dbname = pg.config.Databases[i]
connstr = append(connstr, fmt.Sprintf("host=%s user=%s password=%s dbname=%s sslmode=disable", pg.config.Host, pg.config.Username, pg.config.Password, dbname))
}
return connstr
}
func (pg *PG) getVersion() error {
var version string
result, queryErr := pg.db.Query("show server_version;")
if queryErr != nil {
log.Log.Error(queryErr, "could get postgres version")
return queryErr
}
result.Next()
scanErr := result.Scan(&version)
if scanErr != nil {
log.Log.Error(scanErr, "scan postgres version with error")
return scanErr
}
pg.version = strings.Split(version, " ")[0]
log.Log.Info("get postgres version", "version", version, "instance", pg.config.Name)
return nil
}
func (pg *PG) isVersionAboveV15() bool {
aboveV15 := false
if pg.version != "" {
version, err := strconv.ParseFloat(pg.version, 64)
if err != nil {
log.Log.Error(err, "failed to convert version to number", "version", pg.version)
} else {
if version >= 15.0 {
aboveV15 = true
}
}
}
return aboveV15
}
func (pg *PG) getQuiesceCmd() string {
if pg.isVersionAboveV15() {
return PG_BACKUP_START
}
return PG_START_BACKUP
}
func (pg *PG) getUnQuiesceCmd() string {
if pg.isVersionAboveV15() {
return PG_BACKUP_STOP
}
return PG_STOP_BACKUP
}