build source

This commit is contained in:
build 2026-04-16 04:16:36 +00:00
commit ee1fec43ed
4171 changed files with 1351288 additions and 0 deletions

View file

@ -0,0 +1,125 @@
# Created by https://www.gitignore.io/api/go,intellij+all,visualstudiocode
# Edit at https://www.gitignore.io/?templates=go,intellij+all,visualstudiocode
### Go ###
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Dependency directories (remove the comment below to include it)
# vendor/
### Go Patch ###
/vendor/
/Godeps/
### Intellij+all ###
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
# Generated files
.idea/**/contentModel.xml
# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
# Gradle
.idea/**/gradle.xml
.idea/**/libraries
# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
# .idea/modules.xml
# .idea/*.iml
# .idea/modules
# *.iml
# *.ipr
# CMake
cmake-build-*/
# Mongo Explorer plugin
.idea/**/mongoSettings.xml
# File-based project format
*.iws
# IntelliJ
out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Cursive Clojure plugin
.idea/replstate.xml
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Editor-based Rest Client
.idea/httpRequests
# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser
### Intellij+all Patch ###
# Ignores the whole .idea folder and all .iml files
# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360
.idea/
# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023
*.iml
modules.xml
.idea/misc.xml
*.ipr
# Sonarlint plugin
.idea/sonarlint
### VisualStudioCode ###
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
### VisualStudioCode Patch ###
# Ignore all local history of files
.history
# End of https://www.gitignore.io/api/go,intellij+all,visualstudiocode
main

View file

@ -0,0 +1,55 @@
linters:
disable-all: true
enable:
- deadcode
- errcheck
- gosimple
- govet
- ineffassign
- staticcheck
- structcheck
- typecheck
- unused
- varcheck
- bodyclose
- depguard
- dogsled
- dupl
- funlen
- gochecknoglobals
- gochecknoinits
- gocognit
- goconst
- gocritic
- gocyclo
- godox
- gofmt
- goimports
- revive
- misspell
- nakedret
- prealloc
- exportloopref
- stylecheck
- unconvert
- unparam
- whitespace
- wsl
issues:
exclude-use-default: false
exclude:
- Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*printf?|os\.(Un)?Setenv). is not checked
- ST1000
- func name will be used as test\.Test.* by other packages, and that stutters; consider calling this
- (possible misuse of unsafe.Pointer|should have signature)
- ineffective break statement. Did you mean to break out of the outer loop
- Use of unsafe calls should be audited
- Subprocess launch(ed with variable|ing should be audited)
- G104
- (Expect directory permissions to be 0750 or less|Expect file permissions to be 0600 or less)
- Potential file inclusion via variable
exclude-rules:
- path: '(.+)_test\.go'
linters:
- funlen
- goconst

View file

@ -0,0 +1,22 @@
# Contributing to embedded-postgres
Thank you for taking the time to contribute. These are mostly guidelines, not rules. Use your best judgment, and feel
free to propose changes to this document in a pull request.
# Working with forked go repos
If you haven't worked with forked go repos before, take a look at this blog post for some excellent advice
about [contributing to go open source git repositories](https://splice.com/blog/contributing-open-source-git-repositories-go/)
.
# PRs
- Please open PRs against master.
- We prefer single commit PRs, but sometimes for multiple commits are justified - use your best judgement.
- Please add/modify tests to cover the proposed code changes.
- If the PR contains a new feature, please document it in the README.
# Documentation
For simple typo fixes and documentation improvements feel free to raise a PR without raising an issue in github. For
anything more complicated please file an issue.

View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019 Fergus Strange
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View file

@ -0,0 +1,122 @@
<p align="center">
<img src="https://raw.githubusercontent.com/fergusstrange/embedded-postgres/master/gopher.png" width="150">
</p>
<p align="center">
<a href="https://godoc.org/github.com/fergusstrange/embedded-postgres"><img src="https://godoc.org/github.com/fergusstrange/embedded-postgres?status.svg" alt="Godoc" /></a>
<a href='https://coveralls.io/github/fergusstrange/embedded-postgres?branch=master'><img src='https://coveralls.io/repos/github/fergusstrange/embedded-postgres/badge.svg?branch=master' alt='Coverage Status' /></a>
<a href="https://github.com/fergusstrange/embedded-postgres/actions"><img src="https://github.com/fergusstrange/embedded-postgres/workflows/Embedded%20Postgres/badge.svg" alt="Build Status" /></a>
<a href="https://app.circleci.com/pipelines/github/fergusstrange/embedded-postgres"><img src="https://circleci.com/gh/fergusstrange/embedded-postgres.svg?style=shield" alt="Build Status" /></a>
<a href="https://goreportcard.com/report/github.com/fergusstrange/embedded-postgres"><img src="https://goreportcard.com/badge/github.com/fergusstrange/embedded-postgres" alt="Go Report Card" /></a>
</p>
# embedded-postgres
Run a real Postgres database locally on Linux, OSX or Windows as part of another Go application or test.
When testing this provides a higher level of confidence than using any in memory alternative. It also requires no other
external dependencies outside of the Go build ecosystem.
Heavily inspired by Java projects [zonkyio/embedded-postgres](https://github.com/zonkyio/embedded-postgres)
and [opentable/otj-pg-embedded](https://github.com/opentable/otj-pg-embedded) and reliant on the great work being done
by [zonkyio/embedded-postgres-binaries](https://github.com/zonkyio/embedded-postgres-binaries) in order to fetch
precompiled binaries
from [Maven](https://mvnrepository.com/artifact/io.zonky.test.postgres/embedded-postgres-binaries-bom).
## Installation
embedded-postgres uses Go modules and as such can be referenced by release version for use as a library. Use the
following to add the latest release to your project.
```bash
go get -u github.com/fergusstrange/embedded-postgres
```
Please note that Postgres versions before 18.3.0 on Mac/Darwin require [Rosetta 2](https://github.com/fergusstrange/embedded-postgres/blob/cf5b3570ca7fc727fae6e4874ec08b4818b705b1/.circleci/config.yml#L28) on Apple Silicon due to the upstream binaries being x86_64-only. Version 18.3.0+ includes universal binaries that work natively on Apple Silicon.
## How to use
This library aims to require as little configuration as possible, favouring overridable defaults
| Configuration | Default Value |
|---------------------|-------------------------------------------------|
| Username | postgres |
| Password | postgres |
| Database | postgres |
| Version | 18.3.0 |
| Encoding | UTF8 |
| Locale | C |
| CachePath | $USER_HOME/.embedded-postgres-go/ |
| RuntimePath | $USER_HOME/.embedded-postgres-go/extracted |
| DataPath | $USER_HOME/.embedded-postgres-go/extracted/data |
| BinariesPath | $USER_HOME/.embedded-postgres-go/extracted |
| BinaryRepositoryURL | https://repo1.maven.org/maven2 |
| Port | 5432 |
| StartTimeout | 15 Seconds |
| StartParameters | map[string]string{"max_connections": "101"} |
The *RuntimePath* directory is erased and recreated at each `Start()` and therefore not suitable for persistent data.
If a persistent data location is required, set *DataPath* to a directory outside *RuntimePath*.
If the *RuntimePath* directory is empty or already initialized but with an incompatible postgres version, it will be
removed and Postgres reinitialized.
Postgres binaries will be downloaded and placed in *BinaryPath* if `BinaryPath/bin` doesn't exist.
*BinaryRepositoryURL* parameter allow overriding maven repository url for Postgres binaries.
If the directory does exist, whatever binary version is placed there will be used (no version check
is done).
If your test need to run multiple different versions of Postgres for different tests, make sure
*BinaryPath* is a subdirectory of *RuntimePath*.
A single Postgres instance can be created, started and stopped as follows
```go
postgres := embeddedpostgres.NewDatabase()
err := postgres.Start()
// Do test logic
err := postgres.Stop()
```
or created with custom configuration
```go
logger := &bytes.Buffer{}
postgres := NewDatabase(DefaultConfig().
Username("beer").
Password("wine").
Database("gin").
Version(V12).
RuntimePath("/tmp").
BinaryRepositoryURL("https://repo.local/central.proxy").
Port(9876).
StartTimeout(45 * time.Second).
StartParameters(map[string]string{"max_connections": "200"}).
Logger(logger))
err := postgres.Start()
// Do test logic
err := postgres.Stop()
```
It should be noted that if `postgres.Stop()` is not called then the child Postgres process will not be released and the
caller will block.
## Examples
There are a number of realistic representations of how to use this library
in [examples](https://github.com/fergusstrange/embedded-postgres/tree/master/examples).
## Credits
- [Gopherize Me](https://gopherize.me) Thanks for the awesome logo template.
- [zonkyio/embedded-postgres-binaries](https://github.com/zonkyio/embedded-postgres-binaries) Without which the
precompiled Postgres binaries would not exist for this to work.
## Contributing
View the [contributing guide](CONTRIBUTING.md).

View file

@ -0,0 +1,37 @@
package embeddedpostgres
import (
"fmt"
"os"
"path/filepath"
)
// CacheLocator retrieves the location of the Postgres binary cache returning it to location.
// The result of whether this cache is present will be returned to exists.
type CacheLocator func() (location string, exists bool)
func defaultCacheLocator(cacheDirectory string, versionStrategy VersionStrategy) CacheLocator {
return func() (string, bool) {
if cacheDirectory == "" {
cacheDirectory = ".embedded-postgres-go"
if userHome, err := os.UserHomeDir(); err == nil {
cacheDirectory = filepath.Join(userHome, ".embedded-postgres-go")
}
}
operatingSystem, architecture, version := versionStrategy()
cacheLocation := filepath.Join(cacheDirectory,
fmt.Sprintf("embedded-postgres-binaries-%s-%s-%s.txz",
operatingSystem,
architecture,
version))
info, err := os.Stat(cacheLocation)
if err != nil {
return cacheLocation, os.IsExist(err) && !info.IsDir()
}
return cacheLocation, !info.IsDir()
}
}

View file

@ -0,0 +1,166 @@
package embeddedpostgres
import (
"fmt"
"io"
"os"
"time"
)
// Config maintains the runtime configuration for the Postgres process to be created.
type Config struct {
version PostgresVersion
port uint32
database string
username string
password string
cachePath string
runtimePath string
dataPath string
binariesPath string
locale string
encoding string
startParameters map[string]string
binaryRepositoryURL string
startTimeout time.Duration
logger io.Writer
}
// DefaultConfig provides a default set of configuration to be used "as is" or modified using the provided builders.
// The following can be assumed as defaults:
// Version: 16
// Port: 5432
// Database: postgres
// Username: postgres
// Password: postgres
// StartTimeout: 15 Seconds
func DefaultConfig() Config {
return Config{
version: V18,
port: 5432,
database: "postgres",
username: "postgres",
password: "postgres",
startTimeout: 15 * time.Second,
logger: os.Stdout,
binaryRepositoryURL: "https://repo1.maven.org/maven2",
}
}
// Version will set the Postgres binary version.
func (c Config) Version(version PostgresVersion) Config {
c.version = version
return c
}
// Port sets the runtime port that Postgres can be accessed on.
func (c Config) Port(port uint32) Config {
c.port = port
return c
}
// Database sets the database name that will be created.
func (c Config) Database(database string) Config {
c.database = database
return c
}
// Username sets the username that will be used to connect.
func (c Config) Username(username string) Config {
c.username = username
return c
}
// Password sets the password that will be used to connect.
func (c Config) Password(password string) Config {
c.password = password
return c
}
// RuntimePath sets the path that will be used for the extracted Postgres runtime directory.
// If Postgres data directory is not set with DataPath(), this directory is also used as data directory.
func (c Config) RuntimePath(path string) Config {
c.runtimePath = path
return c
}
// CachePath sets the path that will be used for storing Postgres binaries archive.
// If this option is not set, ~/.go-embedded-postgres will be used.
func (c Config) CachePath(path string) Config {
c.cachePath = path
return c
}
// DataPath sets the path that will be used for the Postgres data directory.
// If this option is set, a previously initialized data directory will be reused if possible.
func (c Config) DataPath(path string) Config {
c.dataPath = path
return c
}
// BinariesPath sets the path of the pre-downloaded postgres binaries.
// If this option is left unset, the binaries will be downloaded.
func (c Config) BinariesPath(path string) Config {
c.binariesPath = path
return c
}
// Locale sets the default locale for initdb
func (c Config) Locale(locale string) Config {
c.locale = locale
return c
}
// Encoding sets the default character set for initdb
func (c Config) Encoding(encoding string) Config {
c.encoding = encoding
return c
}
// StartParameters sets run-time parameters when starting Postgres (passed to Postgres via "-c").
//
// These parameters can be used to override the default configuration values in postgres.conf such
// as max_connections=100. See https://www.postgresql.org/docs/current/runtime-config.html
func (c Config) StartParameters(parameters map[string]string) Config {
c.startParameters = parameters
return c
}
// StartTimeout sets the max timeout that will be used when starting the Postgres process and creating the initial database.
func (c Config) StartTimeout(timeout time.Duration) Config {
c.startTimeout = timeout
return c
}
// Logger sets the logger for postgres output
func (c Config) Logger(logger io.Writer) Config {
c.logger = logger
return c
}
// BinaryRepositoryURL set BinaryRepositoryURL to fetch PG Binary in case of Maven proxy
func (c Config) BinaryRepositoryURL(binaryRepositoryURL string) Config {
c.binaryRepositoryURL = binaryRepositoryURL
return c
}
func (c Config) GetConnectionURL() string {
return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s", c.username, c.password, "localhost", c.port, c.database)
}
// PostgresVersion represents the semantic version used to fetch and run the Postgres process.
type PostgresVersion string
// Predefined supported Postgres versions.
const (
V18 = PostgresVersion("18.3.0")
V17 = PostgresVersion("17.5.0")
V16 = PostgresVersion("16.9.0")
V15 = PostgresVersion("15.13.0")
V14 = PostgresVersion("14.18.0")
V13 = PostgresVersion("13.21.0")
V12 = PostgresVersion("12.22.0")
V11 = PostgresVersion("11.22.0")
V10 = PostgresVersion("10.23.0")
V9 = PostgresVersion("9.6.24")
)

View file

@ -0,0 +1,124 @@
package embeddedpostgres
import (
"archive/tar"
"fmt"
"io"
"os"
"path/filepath"
"github.com/xi2/xz"
)
func defaultTarReader(xzReader *xz.Reader) (func() (*tar.Header, error), func() io.Reader) {
tarReader := tar.NewReader(xzReader)
return func() (*tar.Header, error) {
return tarReader.Next()
}, func() io.Reader {
return tarReader
}
}
func decompressTarXz(tarReader func(*xz.Reader) (func() (*tar.Header, error), func() io.Reader), path, extractPath string) error {
extractDirectory := filepath.Dir(extractPath)
if err := os.MkdirAll(extractDirectory, os.ModePerm); err != nil {
return errorUnableToExtract(path, extractPath, err)
}
tempExtractPath, err := os.MkdirTemp(extractDirectory, "temp_")
if err != nil {
return errorUnableToExtract(path, extractPath, err)
}
defer func() {
if err := os.RemoveAll(tempExtractPath); err != nil {
panic(err)
}
}()
tarFile, err := os.Open(path)
if err != nil {
return errorUnableToExtract(path, extractPath, err)
}
defer func() {
if err := tarFile.Close(); err != nil {
panic(err)
}
}()
xzReader, err := xz.NewReader(tarFile, 0)
if err != nil {
return errorUnableToExtract(path, extractPath, err)
}
readNext, reader := tarReader(xzReader)
for {
header, err := readNext()
if err == io.EOF {
break
}
if err != nil {
return errorExtractingPostgres(err)
}
targetPath := filepath.Join(tempExtractPath, header.Name)
finalPath := filepath.Join(extractPath, header.Name)
if err := os.MkdirAll(filepath.Dir(targetPath), os.ModePerm); err != nil {
return errorExtractingPostgres(err)
}
if err := os.MkdirAll(filepath.Dir(finalPath), os.ModePerm); err != nil {
return errorExtractingPostgres(err)
}
switch header.Typeflag {
case tar.TypeReg:
outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
if err != nil {
return errorExtractingPostgres(err)
}
if _, err := io.Copy(outFile, reader()); err != nil {
return errorExtractingPostgres(err)
}
if err := outFile.Close(); err != nil {
return errorExtractingPostgres(err)
}
case tar.TypeSymlink:
if err := os.RemoveAll(targetPath); err != nil {
return errorExtractingPostgres(err)
}
if err := os.Symlink(header.Linkname, targetPath); err != nil {
return errorExtractingPostgres(err)
}
case tar.TypeDir:
if err := os.MkdirAll(finalPath, os.FileMode(header.Mode)); err != nil {
return errorExtractingPostgres(err)
}
continue
}
if err := renameOrIgnore(targetPath, finalPath); err != nil {
return errorExtractingPostgres(err)
}
}
return nil
}
func errorUnableToExtract(cacheLocation, binariesPath string, err error) error {
return fmt.Errorf("unable to extract postgres archive %s to %s, if running parallel tests, configure RuntimePath to isolate testing directories, %w",
cacheLocation,
binariesPath,
err,
)
}

View file

@ -0,0 +1,268 @@
package embeddedpostgres
import (
"errors"
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"sync"
)
var mu sync.Mutex
var (
ErrServerNotStarted = errors.New("server has not been started")
ErrServerAlreadyStarted = errors.New("server is already started")
)
// EmbeddedPostgres maintains all configuration and runtime functions for maintaining the lifecycle of one Postgres process.
type EmbeddedPostgres struct {
config Config
cacheLocator CacheLocator
remoteFetchStrategy RemoteFetchStrategy
initDatabase initDatabase
createDatabase createDatabase
started bool
syncedLogger *syncedLogger
}
// NewDatabase creates a new EmbeddedPostgres struct that can be used to start and stop a Postgres process.
// When called with no parameters it will assume a default configuration state provided by the DefaultConfig method.
// When called with parameters the first Config parameter will be used for configuration.
func NewDatabase(config ...Config) *EmbeddedPostgres {
if len(config) < 1 {
return newDatabaseWithConfig(DefaultConfig())
}
return newDatabaseWithConfig(config[0])
}
func newDatabaseWithConfig(config Config) *EmbeddedPostgres {
versionStrategy := defaultVersionStrategy(
config,
runtime.GOOS,
runtime.GOARCH,
linuxMachineName,
shouldUseAlpineLinuxBuild,
)
cacheLocator := defaultCacheLocator(config.cachePath, versionStrategy)
remoteFetchStrategy := defaultRemoteFetchStrategy(config.binaryRepositoryURL, versionStrategy, cacheLocator)
return &EmbeddedPostgres{
config: config,
cacheLocator: cacheLocator,
remoteFetchStrategy: remoteFetchStrategy,
initDatabase: defaultInitDatabase,
createDatabase: defaultCreateDatabase,
started: false,
}
}
// Start will try to start the configured Postgres process returning an error when there were any problems with invocation.
// If any error occurs Start will try to also Stop the Postgres process in order to not leave any sub-process running.
//
//nolint:funlen
func (ep *EmbeddedPostgres) Start() error {
if ep.started {
return ErrServerAlreadyStarted
}
if err := ensurePortAvailable(ep.config.port); err != nil {
return err
}
logger, err := newSyncedLogger("", ep.config.logger)
if err != nil {
return errors.New("unable to create logger")
}
ep.syncedLogger = logger
cacheLocation, cacheExists := ep.cacheLocator()
if ep.config.runtimePath == "" {
ep.config.runtimePath = filepath.Join(filepath.Dir(cacheLocation), "extracted")
}
if ep.config.dataPath == "" {
ep.config.dataPath = filepath.Join(ep.config.runtimePath, "data")
}
if err := os.RemoveAll(ep.config.runtimePath); err != nil {
return fmt.Errorf("unable to clean up runtime directory %s with error: %s", ep.config.runtimePath, err)
}
if ep.config.binariesPath == "" {
ep.config.binariesPath = ep.config.runtimePath
}
if err := ep.downloadAndExtractBinary(cacheExists, cacheLocation); err != nil {
return err
}
if err := os.MkdirAll(ep.config.runtimePath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create runtime directory %s with error: %s", ep.config.runtimePath, err)
}
reuseData := dataDirIsValid(ep.config.dataPath, ep.config.version)
if !reuseData {
if err := ep.cleanDataDirectoryAndInit(); err != nil {
return err
}
}
if err := startPostgres(ep); err != nil {
return err
}
if err := ep.syncedLogger.flush(); err != nil {
return err
}
ep.started = true
if !reuseData {
if err := ep.createDatabase(ep.config.port, ep.config.username, ep.config.password, ep.config.database); err != nil {
if stopErr := stopPostgres(ep); stopErr != nil {
return fmt.Errorf("unable to stop database caused by error %s", err)
}
return err
}
}
if err := healthCheckDatabaseOrTimeout(ep.config); err != nil {
if stopErr := stopPostgres(ep); stopErr != nil {
return fmt.Errorf("unable to stop database caused by error %s", err)
}
return err
}
return nil
}
func (ep *EmbeddedPostgres) downloadAndExtractBinary(cacheExists bool, cacheLocation string) error {
// lock to prevent collisions with duplicate downloads
mu.Lock()
defer mu.Unlock()
_, binDirErr := os.Stat(filepath.Join(ep.config.binariesPath, "bin", "pg_ctl"))
if os.IsNotExist(binDirErr) {
if !cacheExists {
if err := ep.remoteFetchStrategy(); err != nil {
return err
}
}
if err := decompressTarXz(defaultTarReader, cacheLocation, ep.config.binariesPath); err != nil {
return err
}
}
return nil
}
func (ep *EmbeddedPostgres) cleanDataDirectoryAndInit() error {
if err := os.RemoveAll(ep.config.dataPath); err != nil {
return fmt.Errorf("unable to clean up data directory %s with error: %s", ep.config.dataPath, err)
}
if err := ep.initDatabase(ep.config.binariesPath, ep.config.runtimePath, ep.config.dataPath, ep.config.username, ep.config.password, ep.config.locale, ep.config.encoding, ep.syncedLogger.file); err != nil {
return err
}
return nil
}
// Stop will try to stop the Postgres process gracefully returning an error when there were any problems.
func (ep *EmbeddedPostgres) Stop() error {
if !ep.started {
return ErrServerNotStarted
}
if err := stopPostgres(ep); err != nil {
return err
}
ep.started = false
if err := ep.syncedLogger.flush(); err != nil {
return err
}
return nil
}
func encodeOptions(port uint32, parameters map[string]string) string {
options := []string{fmt.Sprintf("-p %d", port)}
for k, v := range parameters {
// Double-quote parameter values - they may have spaces.
// Careful: CMD on Windows uses only double quotes to delimit strings.
// It treats single quotes as regular characters.
options = append(options, fmt.Sprintf("-c %s=\"%s\"", k, v))
}
return strings.Join(options, " ")
}
func startPostgres(ep *EmbeddedPostgres) error {
postgresBinary := filepath.Join(ep.config.binariesPath, "bin/pg_ctl")
postgresProcess := exec.Command(postgresBinary, "start", "-w",
"-D", ep.config.dataPath,
"-o", encodeOptions(ep.config.port, ep.config.startParameters))
postgresProcess.Stdout = ep.syncedLogger.file
postgresProcess.Stderr = ep.syncedLogger.file
if err := postgresProcess.Run(); err != nil {
_ = ep.syncedLogger.flush()
logContent, _ := readLogsOrTimeout(ep.syncedLogger.file)
return fmt.Errorf("could not start postgres using %s:\n%s", postgresProcess.String(), string(logContent))
}
return nil
}
func stopPostgres(ep *EmbeddedPostgres) error {
postgresBinary := filepath.Join(ep.config.binariesPath, "bin/pg_ctl")
postgresProcess := exec.Command(postgresBinary, "stop", "-w",
"-D", ep.config.dataPath)
postgresProcess.Stderr = ep.syncedLogger.file
postgresProcess.Stdout = ep.syncedLogger.file
if err := postgresProcess.Run(); err != nil {
return err
}
return nil
}
func ensurePortAvailable(port uint32) error {
conn, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", port))
if err != nil {
return fmt.Errorf("process already listening on port %d", port)
}
if err := conn.Close(); err != nil {
return err
}
return nil
}
func dataDirIsValid(dataDir string, version PostgresVersion) bool {
pgVersion := filepath.Join(dataDir, "PG_VERSION")
d, err := os.ReadFile(pgVersion)
if err != nil {
return false
}
v := strings.TrimSuffix(string(d), "\n")
return strings.HasPrefix(string(version), v)
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 264 KiB

View file

@ -0,0 +1,81 @@
package embeddedpostgres
import (
"fmt"
"io"
"io/ioutil"
"os"
"time"
)
type syncedLogger struct {
offset int64
logger io.Writer
file *os.File
}
func newSyncedLogger(dir string, logger io.Writer) (*syncedLogger, error) {
file, err := os.CreateTemp(dir, "embedded_postgres_log")
if err != nil {
return nil, err
}
s := syncedLogger{
logger: logger,
file: file,
}
return &s, nil
}
func (s *syncedLogger) flush() error {
if s.logger != nil {
file, err := os.Open(s.file.Name())
if err != nil {
return fmt.Errorf("unable to process postgres logs: %s", err)
}
defer func() {
if err := file.Close(); err != nil {
panic(err)
}
}()
if _, err = file.Seek(s.offset, io.SeekStart); err != nil {
return fmt.Errorf("unable to process postgres logs: %s", err)
}
readBytes, err := io.Copy(s.logger, file)
if err != nil {
return fmt.Errorf("unable to process postgres logs: %s", err)
}
s.offset += readBytes
}
return nil
}
func readLogsOrTimeout(logger *os.File) (logContent []byte, err error) {
logContent = []byte("logs could not be read")
logContentChan := make(chan []byte, 1)
errChan := make(chan error, 1)
go func() {
if actualLogContent, err := ioutil.ReadFile(logger.Name()); err == nil {
logContentChan <- actualLogContent
} else {
errChan <- err
}
}()
select {
case logContent = <-logContentChan:
case err = <-errChan:
case <-time.After(10 * time.Second):
err = fmt.Errorf("timed out waiting for logs")
}
return logContent, err
}

View file

@ -0,0 +1,173 @@
package embeddedpostgres
import (
"context"
"database/sql"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"github.com/lib/pq"
)
const (
fmtCloseDBConn = "unable to close database connection: %w"
fmtAfterError = "%v happened after error: %w"
)
type initDatabase func(binaryExtractLocation, runtimePath, pgDataDir, username, password, locale string, encoding string, logger *os.File) error
type createDatabase func(port uint32, username, password, database string) error
func defaultInitDatabase(binaryExtractLocation, runtimePath, pgDataDir, username, password, locale string, encoding string, logger *os.File) error {
passwordFile, err := createPasswordFile(runtimePath, password)
if err != nil {
return err
}
args := []string{
"-A", "password",
"-U", username,
"-D", pgDataDir,
fmt.Sprintf("--pwfile=%s", passwordFile),
}
if locale != "" {
args = append(args, fmt.Sprintf("--locale=%s", locale))
}
if encoding != "" {
args = append(args, fmt.Sprintf("--encoding=%s", encoding))
}
postgresInitDBBinary := filepath.Join(binaryExtractLocation, "bin/initdb")
postgresInitDBProcess := exec.Command(postgresInitDBBinary, args...)
postgresInitDBProcess.Stderr = logger
postgresInitDBProcess.Stdout = logger
if err = postgresInitDBProcess.Run(); err != nil {
logContent, readLogsErr := readLogsOrTimeout(logger) // we want to preserve the original error
if readLogsErr != nil {
logContent = []byte(string(logContent) + " - " + readLogsErr.Error())
}
return fmt.Errorf("unable to init database using '%s': %w\n%s", postgresInitDBProcess.String(), err, string(logContent))
}
if err = os.Remove(passwordFile); err != nil {
return fmt.Errorf("unable to remove password file '%v': %w", passwordFile, err)
}
return nil
}
func createPasswordFile(runtimePath, password string) (string, error) {
passwordFileLocation := filepath.Join(runtimePath, "pwfile")
if err := os.WriteFile(passwordFileLocation, []byte(password), 0600); err != nil {
return "", fmt.Errorf("unable to write password file to %s", passwordFileLocation)
}
return passwordFileLocation, nil
}
func defaultCreateDatabase(port uint32, username, password, database string) (err error) {
if database == "postgres" {
return nil
}
conn, err := openDatabaseConnection(port, username, password, "postgres")
if err != nil {
return errorCustomDatabase(database, err)
}
db := sql.OpenDB(conn)
defer func() {
err = connectionClose(db, err)
}()
if _, err := db.Exec(fmt.Sprintf("CREATE DATABASE \"%s\"", database)); err != nil {
return errorCustomDatabase(database, err)
}
return nil
}
// connectionClose closes the database connection and handles the error of the function that used the database connection
func connectionClose(db io.Closer, err error) error {
closeErr := db.Close()
if closeErr != nil {
closeErr = fmt.Errorf(fmtCloseDBConn, closeErr)
if err != nil {
err = fmt.Errorf(fmtAfterError, closeErr, err)
} else {
err = closeErr
}
}
return err
}
func healthCheckDatabaseOrTimeout(config Config) error {
healthCheckSignal := make(chan bool)
defer close(healthCheckSignal)
timeout, cancelFunc := context.WithTimeout(context.Background(), config.startTimeout)
defer cancelFunc()
go func() {
for timeout.Err() == nil {
if err := healthCheckDatabase(config.port, config.database, config.username, config.password); err != nil {
continue
}
healthCheckSignal <- true
break
}
}()
select {
case <-healthCheckSignal:
return nil
case <-timeout.Done():
return errors.New("timed out waiting for database to become available")
}
}
func healthCheckDatabase(port uint32, database, username, password string) (err error) {
conn, err := openDatabaseConnection(port, username, password, database)
if err != nil {
return err
}
db := sql.OpenDB(conn)
defer func() {
err = connectionClose(db, err)
}()
if _, err := db.Query("SELECT 1"); err != nil {
return err
}
return nil
}
func openDatabaseConnection(port uint32, username string, password string, database string) (*pq.Connector, error) {
conn, err := pq.NewConnector(fmt.Sprintf("host=localhost port=%d user=%s password=%s dbname=%s sslmode=disable",
port,
username,
password,
database))
if err != nil {
return nil, err
}
return conn, nil
}
func errorCustomDatabase(database string, err error) error {
return fmt.Errorf("unable to connect to create database with custom name %s with the following error: %s", database, err)
}

View file

@ -0,0 +1,169 @@
package embeddedpostgres
import (
"archive/zip"
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strings"
)
// RemoteFetchStrategy provides a strategy to fetch a Postgres binary so that it is available for use.
type RemoteFetchStrategy func() error
//nolint:funlen
func defaultRemoteFetchStrategy(remoteFetchHost string, versionStrategy VersionStrategy, cacheLocator CacheLocator) RemoteFetchStrategy {
return func() error {
operatingSystem, architecture, version := versionStrategy()
jarDownloadURL := fmt.Sprintf("%s/io/zonky/test/postgres/embedded-postgres-binaries-%s-%s/%s/embedded-postgres-binaries-%s-%s-%s.jar",
remoteFetchHost,
operatingSystem,
architecture,
version,
operatingSystem,
architecture,
version)
jarDownloadResponse, err := http.Get(jarDownloadURL)
if err != nil {
return fmt.Errorf("unable to connect to %s", remoteFetchHost)
}
defer closeBody(jarDownloadResponse)()
if jarDownloadResponse.StatusCode != http.StatusOK {
return fmt.Errorf("no version found matching %s", version)
}
jarBodyBytes, err := io.ReadAll(jarDownloadResponse.Body)
if err != nil {
return errorFetchingPostgres(err)
}
shaDownloadURL := fmt.Sprintf("%s.sha256", jarDownloadURL)
shaDownloadResponse, err := http.Get(shaDownloadURL)
if err != nil {
return fmt.Errorf("download sha256 from %s failed: %w", shaDownloadURL, err)
}
defer closeBody(shaDownloadResponse)()
if err == nil && shaDownloadResponse.StatusCode == http.StatusOK {
if shaBodyBytes, err := io.ReadAll(shaDownloadResponse.Body); err == nil {
jarChecksum := sha256.Sum256(jarBodyBytes)
if !bytes.Equal(shaBodyBytes, []byte(hex.EncodeToString(jarChecksum[:]))) {
return errors.New("downloaded checksums do not match")
}
}
}
return decompressResponse(jarBodyBytes, jarDownloadResponse.ContentLength, cacheLocator, jarDownloadURL)
}
}
func closeBody(resp *http.Response) func() {
return func() {
if resp == nil || resp.Body == nil {
return
}
if err := resp.Body.Close(); err != nil {
log.Fatal(err)
}
}
}
func decompressResponse(bodyBytes []byte, contentLength int64, cacheLocator CacheLocator, downloadURL string) error {
size := contentLength
// if the content length is not set (i.e. chunked encoding),
// we need to use the length of the bodyBytes otherwise
// the unzip operation will fail
if contentLength < 0 {
size = int64(len(bodyBytes))
}
zipReader, err := zip.NewReader(bytes.NewReader(bodyBytes), size)
if err != nil {
return errorFetchingPostgres(err)
}
cacheLocation, _ := cacheLocator()
if err := os.MkdirAll(filepath.Dir(cacheLocation), 0755); err != nil {
return errorExtractingPostgres(err)
}
for _, file := range zipReader.File {
if !file.FileHeader.FileInfo().IsDir() && strings.HasSuffix(file.FileHeader.Name, ".txz") {
if err := decompressSingleFile(file, cacheLocation); err != nil {
return err
}
// we have successfully found the file, return early
return nil
}
}
return fmt.Errorf("error fetching postgres: cannot find binary in archive retrieved from %s", downloadURL)
}
func decompressSingleFile(file *zip.File, cacheLocation string) error {
renamed := false
archiveReader, err := file.Open()
if err != nil {
return errorExtractingPostgres(err)
}
archiveBytes, err := io.ReadAll(archiveReader)
if err != nil {
return errorExtractingPostgres(err)
}
// if multiple processes attempt to extract
// to prevent file corruption when multiple processes attempt to extract at the same time
// first to a cache location, and then move the file into place.
tmp, err := os.CreateTemp(filepath.Dir(cacheLocation), "temp_")
if err != nil {
return errorExtractingPostgres(err)
}
defer func() {
// if anything failed before the rename then the temporary file should be cleaned up.
// if the rename was successful then there is no temporary file to remove.
if !renamed {
if err := os.Remove(tmp.Name()); err != nil {
panic(err)
}
}
}()
if _, err := tmp.Write(archiveBytes); err != nil {
return errorExtractingPostgres(err)
}
// Windows cannot rename a file if is it still open.
// The file needs to be manually closed to allow the rename to happen
if err := tmp.Close(); err != nil {
return errorExtractingPostgres(err)
}
if err := renameOrIgnore(tmp.Name(), cacheLocation); err != nil {
return errorExtractingPostgres(err)
}
renamed = true
return nil
}
func errorExtractingPostgres(err error) error {
return fmt.Errorf("unable to extract postgres archive: %s", err)
}
func errorFetchingPostgres(err error) error {
return fmt.Errorf("error fetching postgres: %s", err)
}

View file

@ -0,0 +1,27 @@
package embeddedpostgres
import (
"errors"
"os"
"syscall"
)
// renameOrIgnore will rename the oldpath to the newpath.
//
// On Unix this will be a safe atomic operation.
// On Windows this will do nothing if the new path already exists.
//
// This is only safe to use if you can be sure that the newpath is either missing, or contains the same data as the
// old path.
func renameOrIgnore(oldpath, newpath string) error {
err := os.Rename(oldpath, newpath)
// if the error is due to syscall.EEXIST then this is most likely windows, and a race condition with
// multiple downloads of the file. We can assume that the existing file is the correct one and ignore
// the error
if errors.Is(err, syscall.EEXIST) {
return nil
}
return err
}

View file

@ -0,0 +1,12 @@
package embeddedpostgres
import "testing"
func TestGetConnectionURL(t *testing.T) {
config := DefaultConfig().Database("mydb").Username("myuser").Password("mypass")
expect := "postgresql://myuser:mypass@localhost:5432/mydb"
if got := config.GetConnectionURL(); got != expect {
t.Errorf("expected \"%s\" got \"%s\"", expect, got)
}
}

View file

@ -0,0 +1,68 @@
package embeddedpostgres
import (
"fmt"
"os"
"os/exec"
"strings"
)
// VersionStrategy provides a strategy that can be used to determine which version of Postgres should be used based on
// the operating system, architecture and desired Postgres version.
type VersionStrategy func() (operatingSystem string, architecture string, postgresVersion PostgresVersion)
func defaultVersionStrategy(config Config, goos, arch string, linuxMachineName func() string, shouldUseAlpineLinuxBuild func() bool) VersionStrategy {
return func() (string, string, PostgresVersion) {
goos := goos
arch := arch
if goos == "linux" {
// the zonkyio/embedded-postgres-binaries project produces
// arm binaries with the following name schema:
// 32bit: arm32v6 / arm32v7
// 64bit (aarch64): arm64v8
if arch == "arm64" {
arch += "v8"
} else if arch == "arm" {
machineName := linuxMachineName()
if strings.HasPrefix(machineName, "armv7") {
arch += "32v7"
} else if strings.HasPrefix(machineName, "armv6") {
arch += "32v6"
}
}
if shouldUseAlpineLinuxBuild() {
arch += "-alpine"
}
}
// postgres below version 14.2 is not available for macos on arm
if goos == "darwin" && arch == "arm64" {
var majorVer, minorVer int
if _, err := fmt.Sscanf(string(config.version), "%d.%d", &majorVer, &minorVer); err == nil &&
(majorVer < 14 || (majorVer == 14 && minorVer < 2)) {
arch = "amd64"
} else {
arch += "v8"
}
}
return goos, arch, config.version
}
}
func linuxMachineName() string {
var uname string
if output, err := exec.Command("uname", "-m").Output(); err == nil {
uname = string(output)
}
return uname
}
func shouldUseAlpineLinuxBuild() bool {
_, err := os.Stat("/etc/alpine-release")
return err == nil
}