backups

package
v0.0.0-...-b2da2e6 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 16, 2026 License: AGPL-3.0 Imports: 27 Imported by: 4

Documentation

Index

Constants

View Source
const (
	// FilenamePrefix is the prefix used for backup archive files.
	FilenamePrefix = "juju-backup-"

	// FilenameTemplate is used with time.Time.Format to generate a filename.
	FilenameTemplate = FilenamePrefix + "20060102-150405.tar.gz"
)
View Source
const ChecksumFormatSHA256 = "SHA-256"

ChecksumFormatSHA256 identifies how to interpret the checksum for a backup generated with this version of juju. The encoding is not restated here because the recorded checksum carries it: see ChecksumSHA256.

View Source
const UnknownString = "<unknown>"

UnknownString is a marker value for string fields with unknown values.

Variables

View Source
var CheckSpaceFor = func(dir string, expectedSize int64) error {
	total, err := diskTotal(dir)
	if err != nil {
		return errors.Capture(err)
	}
	diskSizeMargin := float64(total) * 0.10
	if diskSizeMargin > minFreeAbsolute {
		diskSizeMargin = minFreeAbsolute
	}
	backupSizeMargin := float64(expectedSize) * 0.20
	if backupSizeMargin < diskSizeMargin {
		backupSizeMargin = diskSizeMargin
	}
	wantFree := uint64(expectedSize) + uint64(backupSizeMargin)

	available, err := diskFree(dir)
	if err != nil {
		return errors.Capture(err)
	}
	if available < wantFree {
		return errors.Errorf("not enough free space in %q; want %dMiB, have %dMiB",
			dir, wantFree/miByte, available/miByte)
	}
	return nil
}

CheckSpaceFor errors when the free space in dir is less than the expected archive size plus a safety margin. The margin is the larger of the smaller of 5GiB or 10% of the total disk size, and 20% of the expected size. It is a variable so tests can trigger the insufficient-space failure, mirroring Create.

View Source
var Create = func(ctx context.Context, meta *Metadata, args CreateArgs) (string, error) {
	if err := ctx.Err(); err != nil {
		return "", err
	}
	if args.Clock == nil {
		return "", errors.New("missing clock")
	}
	if err := checkDestinationDir(args.DestinationDir); err != nil {
		return "", errors.Capture(err)
	}
	for _, entry := range args.DumpEntries {
		if err := checkDumpEntryName(entry.Name); err != nil {
			return "", errors.Capture(err)
		}
	}
	if err := checkObjectEntries(args.ObjectEntries, args.DataDir); err != nil {
		return "", errors.Capture(err)
	}

	stagingDir, err := os.MkdirTemp(args.DestinationDir, tempPrefix)
	if err != nil {
		return "", errors.Errorf("making backups staging directory: %w", err)
	}

	defer func() { _ = os.RemoveAll(stagingDir) }()

	archivePaths := NewNonCanonicalArchivePaths(stagingDir)

	if err := os.MkdirAll(archivePaths.DBDumpDir, 0700); err != nil {
		return "", errors.Errorf("creating temp directories: %w", err)
	}

	meta.Finished = new(args.Clock.Now())
	metadataReader, err := meta.AsJSONBuffer()
	if err != nil {
		return "", errors.Errorf("preparing the metadata: %w", err)
	}
	canonicalPaths := NewCanonicalArchivePaths()
	metaSize, metaSum, err := writeAll(archivePaths.MetadataFile, metadataReader)
	if err != nil {
		return "", errors.Capture(err)
	}
	manifestEntries := []ManifestEntry{{
		Path:   canonicalPaths.MetadataFile,
		Kind:   ManifestKindMetadata,
		Size:   metaSize,
		SHA256: metaSum,
	}}

	filesSize, filesSum, err := buildFilesArchive(ctx,
		archivePaths.FilesArchive, args.FilesToBackUp, args.DataDir, args.ObjectEntries)
	if err != nil {
		return "", errors.Capture(err)
	}
	manifestEntries = append(manifestEntries, ManifestEntry{
		Path:   canonicalPaths.FilesArchive,
		Kind:   ManifestKindFilesArchive,
		Size:   filesSize,
		SHA256: filesSum,
	})

	entries := make([]DumpEntry, len(args.DumpEntries))
	for i, entry := range args.DumpEntries {
		entries[i] = DumpEntry{Name: entry.Name, Reader: contextReader{ctx: ctx, Reader: entry.Reader}}
	}
	dumpEntries, err := buildDump(
		archivePaths.DBDumpDir, canonicalPaths.DBDumpDir, entries)
	if err != nil {
		return "", errors.Capture(err)
	}
	manifestEntries = append(manifestEntries, dumpEntries...)

	manifestReader, err := NewManifest(manifestEntries).AsJSONBuffer()
	if err != nil {
		return "", errors.Errorf("preparing the manifest: %w", err)
	}
	if _, _, err := writeAll(archivePaths.ManifestFile, manifestReader); err != nil {
		return "", errors.Capture(err)
	}

	filename := filepath.Join(args.DestinationDir,
		meta.Started.Format(FilenameTemplate))
	size, checksum, err := buildArchiveAndChecksum(ctx, filename, stagingDir,
		archivePaths.ContentDir)
	if err != nil {
		return "", errors.Capture(err)
	}

	if err := meta.MarkComplete(size, checksum, args.Clock.Now()); err != nil {
		return "", discardArchive(filename,
			errors.Errorf("updating metadata: %w", err))
	}
	if err := ctx.Err(); err != nil {
		return "", discardArchive(filename, err)
	}

	return filename, nil
}

Create builds a new backup archive file in args.DestinationDir, named after meta.Started using FilenameTemplate. It updates the metadata with the file info and returns the archive filename. It is a variable so tests can replace the archive creation with a stub, mirroring GetFilesToBackUp.

View Source
var GetFilesToBackUp = func(rootDir string, paths *Paths) ([]string, error) {
	glob := filepath.Join(rootDir, paths.DataDir, agentsDir, agentsConfs)
	agentConfs, err := filepath.Glob(glob)
	if err != nil {
		return nil, errors.Errorf("failed to fetch agent config files: %w", err)
	}

	glob = filepath.Join(rootDir, paths.DataDir, initDir, "*")
	serviceConfs, err := filepath.Glob(glob)
	if err != nil {
		return nil, errors.Errorf("failed to fetch service config files: %w", err)
	}

	backupFiles := []string{
		filepath.Join(rootDir, paths.DataDir, toolsDir),
	}

	backupFiles = append(backupFiles, agentConfs...)
	backupFiles = append(backupFiles, serviceConfs...)

	sshAuthKeysPath := filepath.Join(rootDir, SSHDir, authKeysFile)
	optional := []string{
		filepath.Join(rootDir, paths.DataDir, sshIdentFile),
		filepath.Join(rootDir, paths.DataDir, serverPEM),
		filepath.Join(rootDir, paths.DataDir, dbSecret),
		filepath.Join(rootDir, paths.DataDir, nonceFile),
		sshAuthKeysPath,
	}
	for _, file := range optional {
		if _, err := os.Stat(file); err != nil {
			if errors.Is(err, os.ErrNotExist) {
				continue
			}
			if errors.Is(err, os.ErrPermission) && file == sshAuthKeysPath {
				continue
			}
			return nil, errors.Capture(err)
		}
		backupFiles = append(backupFiles, file)
	}

	// The object store directory is intentionally not collected here:
	// object contents reach the archive through the object entries
	// derived from the database dumps, which works for both the file
	// and S3 backends, so a backup does not depend on a local cache of
	// S3 objects.
	var finalBackupFiles []string
	for _, file := range backupFiles {
		err := filepath.Walk(file,
			func(path string, info fs.FileInfo, err error) error {
				if err != nil {
					return err
				}
				if info.IsDir() {
					return nil
				}
				if info.Mode().IsRegular() {
					finalBackupFiles = append(finalBackupFiles, path)
				} else if info.Mode()&os.ModeSymlink != 0 {

					if _, err := os.Stat(path); err != nil {
						if errors.Is(err, os.ErrNotExist) {
							return nil
						}
						return err
					}
					finalBackupFiles = append(finalBackupFiles, path)
				}
				return nil
			})
		if err != nil {
			return nil, errors.Errorf("cannot walk %q: %w", file, err)
		}
	}
	return finalBackupFiles, nil
}

GetFilesToBackUp returns the paths that should be included in the backup archive. It is a variable so tests can replace the filesystem walk with a deterministic list, mirroring SSHDir.

View Source
var SSHDir = "/home/ubuntu/.ssh"

SSHDir is the host directory holding the ubuntu user's SSH files. It is a variable so tests can redirect the lookup into their own tree: on hosts where the directory exists but is not readable by the test user, the fatal stat failure would make results depend on the test runner.

View Source
var StageDumps = func(ctx context.Context, dir string, dumps []NamedDump) (*DumpStaging, error) {
	stagingDir, err := os.MkdirTemp(dir, "juju-backup-dump-")
	if err != nil {
		return nil, errors.Capture(err)
	}

	staging := &DumpStaging{dir: stagingDir}

	defer func() {
		if staging != nil {

			_ = staging.Close()
		}
	}()

	for _, dump := range dumps {
		file, err := stageDump(ctx, stagingDir, dump.Name, dump.Export)
		if err != nil {
			return nil, err
		}
		staging.files = append(staging.files, file)
		staging.entries = append(staging.entries, DumpEntry{
			Name:   dump.Name,
			Reader: file,
		})
	}

	result := staging
	staging = nil
	return result, nil
}

StageDumps marshals each dump into a temporary directory inside dir and returns the staging handle holding the resulting entries. The caller must create dir beforehand and must keep the handle open until the dumps have been archived, then Close it. It is a variable so tests can wrap the staging, mirroring Create.

View Source
var UnknownInt64 = int64(math.MaxInt64)

UnknownInt64 is a marker value for int64 fields with unknown values.

View Source
var UnknownVersion = semversion.MustParse("9999.9999.9999")

UnknownVersion is a marker value for version fields with unknown values.

Functions

func BackupDirToUse

func BackupDirToUse(configuredDir string) string

BackupDirToUse returns the desired backup staging dir.

func ChecksumSHA256

func ChecksumSHA256(hexDigest string) string

ChecksumSHA256 returns the canonical recorded form of a hex SHA-256 digest: the algorithm prefix followed by the digest. A digest already carrying the prefix is returned unchanged.

func WalkDumps

func WalkDumps(ctx context.Context, dir string, fn func(name string, reader io.Reader) error) error

WalkDumps reads every staged dump back from dir, passing each file's archive-relative name and contents to fn. It is the read side of StageDumps, used when recovering a backup.

The fn callback receives each file as a reader that will be closed when fn returns. The callback must fully consume the reader before returning; any data left unread is lost because the underlying file is closed immediately after fn returns. Storing the reader for later use (e.g. in a goroutine) is not safe.

Types

type ArchiveData

type ArchiveData struct {
	ArchivePaths
	// contains filtered or unexported fields
}

ArchiveData is a wrapper around a the uncompressed data in a backup archive file. It provides access to the content of the archive. While ArchiveData provides useful functionality, it may not be appropriate for large archives. The contents of the archive are kept in-memory, so large archives could be too taxing on the host. In that case consider using ArchiveWorkspace instead.

func NewArchiveData

func NewArchiveData(data []byte) *ArchiveData

NewArchiveData builds a new archive data wrapper for the given uncompressed data.

func NewArchiveDataReader

func NewArchiveDataReader(r io.Reader) (*ArchiveData, error)

NewArchiveDataReader returns a new archive data wrapper for the data in the provided reader. Note that the entire archive will be read into memory and kept there. So for relatively large archives it will often be more appropriate to use ArchiveWorkspace instead.

func (*ArchiveData) Metadata

func (ad *ArchiveData) Metadata() (*Metadata, error)

Metadata returns the metadata stored in the backup archive. If no metadata is there, errors.NotFound is returned.

func (*ArchiveData) NewBuffer

func (ad *ArchiveData) NewBuffer() *bytes.Buffer

NewBuffer wraps the archive data in a Buffer.

func (*ArchiveData) Version

func (ad *ArchiveData) Version() (*semversion.Number, error)

Version returns the juju version under which the backup archive was created. If no version is found in the archive, it must come from before backup archives included the version. In that case we return version 1.20.

type ArchivePaths

type ArchivePaths struct {
	// ContentDir is the path to the directory within the archive
	// containing all the contents. It is the only file or directory at
	// the top-level of the archive and everything else in the archive
	// is contained in the content directory.
	ContentDir string

	// FilesArchive is the path to the tar file inside the archive
	// containing all the database related files (with the exception of the
	// DB dump files) gathered in by the backup machinery.
	FilesArchive string

	// DBDumpDir is the path to the directory within the archive
	// contents that contains all the files dumped from the juju
	// database.
	DBDumpDir string

	// MetadataFile is the path to the metadata file.
	MetadataFile string

	// ManifestFile is the path to the content manifest: the index of
	// the archive's components with their sizes and SHA-256 hashes.
	ManifestFile string
}

ArchivePaths holds the paths to the files and directories in a backup archive.

func NewCanonicalArchivePaths

func NewCanonicalArchivePaths() ArchivePaths

NewCanonicalArchivePaths composes a new ArchivePaths with default values set. These values are relative (un-rooted) and the canonical slash ("/") is the path separator. Thus the paths are suitable for resolving the paths in a backup archive file (which is a tar file).

func NewNonCanonicalArchivePaths

func NewNonCanonicalArchivePaths(rootDir string) ArchivePaths

NewNonCanonicalArchivePaths builds a new ArchivePaths using default values, rooted at the provided rootDir. The path separator used is platform-dependent. The resulting paths are suitable for locating backup archive contents in a directory into which an archive has been unpacked.

type ArchiveWorkspace

type ArchiveWorkspace struct {
	ArchivePaths
	RootDir string
}

ArchiveWorkspace is a wrapper around backup archive info that has a concrete root directory and an archive unpacked in it.

func NewArchiveWorkspaceReader

func NewArchiveWorkspaceReader(archive io.Reader) (*ArchiveWorkspace, error)

NewArchiveWorkspaceReader returns a new archive workspace with a new workspace dir populated from the archive. Note that this involves unpacking the entire archive into a directory under the host's "temporary" directory. For relatively large archives this could have adverse effects on hosts with little disk space.

func (*ArchiveWorkspace) Close

func (ws *ArchiveWorkspace) Close() error

Close cleans up the workspace dir.

func (*ArchiveWorkspace) Metadata

func (ws *ArchiveWorkspace) Metadata() (*Metadata, error)

Metadata returns the metadata derived from the JSON file in the archive.

func (*ArchiveWorkspace) OpenArchivedFile

func (ws *ArchiveWorkspace) OpenArchivedFile(filename string) (io.Reader, error)

OpenArchivedFile returns an open ReadCloser for the corresponding file in root.tar.

func (*ArchiveWorkspace) UnpackFilesArchive

func (ws *ArchiveWorkspace) UnpackFilesArchive(targetRoot string) error

UnpackFilesArchive extracts root.tar into the target directory.

type BackupCreator

type BackupCreator interface {
	// Create creates an archive with the supplied notes. It honours context
	// cancellation and returns only after generation and lease maintenance stop.
	// On success, the caller must invoke cleanup after consuming the archive.
	// On error, the implementation removes any partial or completed archive.
	Create(ctx context.Context, notes string) (meta *Metadata, archive string, cleanup func(), err error)
}

type ControllerMetadata

type ControllerMetadata struct {
	// UUID contains the controller UUID.
	UUID string

	// MachineID contains controller machine id from which this backup is taken.
	MachineID string

	// MachineInstanceID contains controller machine's instance id from which this backup is taken.
	MachineInstanceID string

	// HANodes contains the number of nodes in this controller's HA configuration.
	HANodes int64
}

ControllerMetadata contains controller specific metadata.

func UnknownController

func UnknownController() ControllerMetadata

UnknownController returns a new backups origin with unknown values.

type CreateArgs

type CreateArgs struct {
	// DestinationDir is the absolute path to the directory in which
	// the archive is stored. The staging area is created there too.
	DestinationDir string

	// DataDir is the absolute path to the source controller's data
	// directory; object entries are stored relative to it. Required
	// when ObjectEntries is not empty.
	DataDir string

	// FilesToBackUp is the list of absolute paths to the files to
	// include in the archive's root.tar.
	FilesToBackUp []string

	// ObjectEntries are the object store objects archived into the
	// archive's root.tar, streamed and validated one at a time.
	ObjectEntries []ObjectEntry

	// DumpEntries are the database dumps written under the archive's
	// dump directory.
	DumpEntries []DumpEntry

	// Clock provides the time stamping the metadata when the archive
	// creation finishes. It must not be nil.
	Clock clock.Clock
}

CreateArgs holds the arguments for building a backup archive.

type DumpEntry

type DumpEntry struct {
	// Name is the path of the entry relative to the archive's dump
	// directory, e.g. "controller.yaml" or "models/<uuid>.yaml".
	Name string

	// Reader provides the contents of the dump entry.
	Reader io.Reader
}

DumpEntry is a single database dump file to include in the backup archive, under the archive's dump directory.

type DumpExportFunc

type DumpExportFunc func(ctx context.Context, writer io.Writer) error

DumpExportFunc marshals one database dump into writer. Implementations must stream (e.g. yaml.Encoder), so the dump's bytes never accumulate in memory.

func YAMLDump

func YAMLDump(v any) DumpExportFunc

YAMLDump returns a DumpExportFunc that streams v as YAML into the writer.

type DumpStaging

type DumpStaging struct {
	// contains filtered or unexported fields
}

DumpStaging is a directory of staged database dumps. Its files feed directly into Create as DumpEntry readers. Close must be called once the dumps have been archived; it releases the readers and removes the staging directory, so a failed archive creation leaves no partial dumps behind.

func (*DumpStaging) Close

func (s *DumpStaging) Close() error

Close closes the staged dump readers and removes the staging directory. Close and remove failures are returned rather than swallowed, but the caller is expected to invoke Close from a deferred cleanup path and only log the error: it must not mask the result of the operation Close is cleaning up after.

func (*DumpStaging) Entries

func (s *DumpStaging) Entries() []DumpEntry

Entries returns the dump entries to pass to Create.

func (*DumpStaging) EntryExceedsLimit

func (s *DumpStaging) EntryExceedsLimit(limit int64) (name string, size int64, err error)

EntryExceedsLimit reports the staged dump exceeding limit, if any. Recovery refuses dumps larger than its own read bound, so a dump that would back up fine but can never be recovered fails the backup instead. The entry name is empty when nothing exceeds the limit.

func (*DumpStaging) Size

func (s *DumpStaging) Size() (int64, error)

Size returns the total size in bytes of the staged dumps.

type Manifest

type Manifest struct {
	FormatVersion int64           `json:"format-version"`
	Files         []ManifestEntry `json:"files"`
}

Manifest is the content index of a backup archive. Unlike metadata.json — the archive's provenance record — the manifest inventories the archive's components with their sizes and content hashes, so a reader can verify each component independently and localise corruption without trusting the archive as a whole.

The manifest lives inside the archive: it inherits the archive's trust level and can never carry the outer archive checksum, which remains the operator-supplied trust anchor.

func NewManifest

func NewManifest(entries []ManifestEntry) *Manifest

NewManifest builds the manifest for the given entries.

func NewManifestJSONReader

func NewManifestJSONReader(in io.Reader) (*Manifest, error)

NewManifestJSONReader extracts a manifest from its JSON file. Only the current format version is decoded: a newer manifest is rejected rather than silently read with the wrong semantics.

func (*Manifest) AsJSONBuffer

func (m *Manifest) AsJSONBuffer() (io.Reader, error)

AsJSONBuffer returns a bytes.Buffer containing the JSON-ified manifest. TODO: reuse the json.Encoder and reset the buffer between calls.

type ManifestEntry

type ManifestEntry struct {
	Path      string            `json:"path"`
	Kind      ManifestEntryKind `json:"kind"`
	Size      int64             `json:"size"`
	SHA256    string            `json:"sha256"`
	ModelUUID string            `json:"model-uuid,omitempty"`
}

ManifestEntry records one archive component: its path inside the archive, what it is, and its uncompressed size and SHA-256 content hash, hex encoded.

type ManifestEntryKind

type ManifestEntryKind string

ManifestEntryKind classifies the archive component a manifest entry records.

const (
	// ManifestKindMetadata is the archive's metadata.json.
	ManifestKindMetadata ManifestEntryKind = "metadata"
	// ManifestKindFilesArchive is the root.tar archive of data-directory
	// files.
	ManifestKindFilesArchive ManifestEntryKind = "files-archive"
	// ManifestKindControllerDump is the controller database dump.
	ManifestKindControllerDump ManifestEntryKind = "controller-dump"
	// ManifestKindModelDump is a model database dump.
	ManifestKindModelDump ManifestEntryKind = "model-dump"
)

func ClassifyManifestPath

func ClassifyManifestPath(archivePath string) (ManifestEntryKind, string, error)

ClassifyManifestPath returns the manifest kind for a canonical archive path, and the model UUID for model dumps. Any path outside the known archive components is an error: the archive carries nothing the writer cannot account for.

type Metadata

type Metadata struct {
	*filestorage.FileMetadata

	// Started records when the backup was started.
	Started time.Time

	// Finished records when the backup was complete.
	Finished *time.Time

	// Origin identifies where the backup was created.
	Origin Origin

	// Notes is an optional user-supplied annotation.
	Notes string

	// FormatVersion stores format version of these metadata.
	FormatVersion int64

	// Controller contains metadata about the controller where the backup was taken.
	Controller ControllerMetadata
}

Metadata contains the metadata for a single backup archive.

func BuildMetadata

func BuildMetadata(file *os.File) (*Metadata, error)

BuildMetadata generates the metadata for a backup archive file.

func NewMetadata

func NewMetadata(now time.Time) *Metadata

NewMetadata returns a new Metadata for a backup archive, in the most current format, started at the given time.

func NewMetadataJSONReader

func NewMetadataJSONReader(in io.Reader) (*Metadata, error)

NewMetadataJSONReader extracts a new metadata from the JSON file.

func (*Metadata) AsJSONBuffer

func (m *Metadata) AsJSONBuffer() (io.Reader, error)

AsJSONBuffer returns a bytes.Buffer containing the JSON-ified metadata. This will always produce latest known format.

func (*Metadata) MarkComplete

func (m *Metadata) MarkComplete(size int64, checksum string, finished time.Time) error

MarkComplete populates the remaining metadata values, finishing at the given time. The default checksum format is used.

type NamedDump

type NamedDump struct {
	Name   string
	Export DumpExportFunc
}

NamedDump pairs a dump entry name with the function producing its contents, e.g. "controller.yaml" or "models/<uuid>.yaml".

type ObjectEntry

type ObjectEntry struct {
	// Namespace is the object store namespace the object belongs to:
	// "controller" for the controller database, or a model UUID.
	Namespace string

	// SHA256 is the hex-encoded SHA-256 of the object, as exported by
	// the database. It identifies the object for reading.
	SHA256 string

	// SHA384 is the hex-encoded SHA-384 of the object, as exported by
	// the database. It names the object inside the archive.
	SHA384 string

	// Size is the expected object size in bytes.
	Size int64

	// Source opens the object for reading.
	Source ObjectSource
}

ObjectEntry is a single object store object to include in the backup's root.tar archive.

type ObjectSource

type ObjectSource func(ctx context.Context) (io.ReadCloser, error)

ObjectSource opens the content of one object store object for reading. The returned reader must be closed by the caller.

type Origin

type Origin struct {
	Model    string
	Machine  string
	Hostname string
	Version  semversion.Number
	Base     string
}

Origin identifies where a backup archive came from. While it is more about where and Metadata about what and when, that distinction does not merit special consideration. Instead, Origin exists separately from Metadata due to its use as an argument when requesting the creation of a new backup.

func UnknownOrigin

func UnknownOrigin() Origin

UnknownOrigin returns a new backups origin with unknown values.

type Paths

type Paths struct {
	BackupDir string
	DataDir   string
	LogsDir   string
}

Paths holds the paths that backups needs.

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址